github-to-sqlite get command, refs #50

Author: simonw

README.md

@@ -166,3 +166,23 @@ If you add the `--fetch` option the command will also fetch the binary content o
     [########----------------------------]  397/1799   22%  00:03:43
 
 You can then use the [datasette-render-images](https://github.com/simonw/datasette-render-images) plugin to browse them visually.
+
+## Making authenticated API calls
+
+The `github-to-sqlite get` command provides a convenient shortcut for making authenticated calls to the API. Once you have created your `auth.json` file (or set a `GITHUB_TOKEN` environment variable) you can use it like this:
+
+    $ github-to-sqlite get https://api.github.com/gists
+
+This will make an authenticated call to the URL you provide and pretty-print the resulting JSON to the console.
+
+You can ommit the `https://api.github.com/` prefix, for example:
+
+    $ github-to-sqlite get /gists
+
+Many GitHub APIs are [paginated using the HTTP Link header](https://docs.github.com/en/rest/guides/traversing-with-pagination). You can follow this pagination and output a list of all of the resulting items using `--paginate`:
+
+    $ github-to-sqlite get /users/simonw/repos --paginate
+
+You can outline newline-delimited JSON for each item using `--nl`. This can be useful for streaming items into another tool.
+
+    $ github-to-sqlite get /users/simonw/repos --nl

github_to_sqlite/cli.py

@@ -1,6 +1,7 @@
 import click
 import datetime
 import pathlib
+import textwrap
 import os
 import sqlite_utils
 import time
@@ -442,6 +443,52 @@ def emojis(db_path, auth, fetch):
                 table.update(emoji["name"], {"image": utils.fetch_image(emoji["url"])})
 
 
+@cli.command()
+@click.argument("url", type=str)
+@click.option(
+    "-a",
+    "--auth",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=True),
+    default="auth.json",
+    help="Path to auth.json token file",
+)
+@click.option(
+    "--paginate",
+    is_flag=True,
+    help="Paginate through all results",
+)
+@click.option(
+    "--nl",
+    is_flag=True,
+    help="Output newline-delimited JSON",
+)
+def get(url, auth, paginate, nl):
+    "Save repos owened by the specified (or authenticated) username or organization"
+    token = load_token(auth)
+    if paginate or nl:
+        first = True
+        while url:
+            response = utils.get(url, token)
+            items = response.json()
+            if first and not nl:
+                click.echo("[")
+            for item in items:
+                if not first and not nl:
+                    click.echo(",")
+                first = False
+                if not nl:
+                    to_dump = json.dumps(item, indent=4)
+                    click.echo(textwrap.indent(to_dump, "    "), nl=False)
+                else:
+                    click.echo(json.dumps(item))
+            url = response.links.get("next", {}).get("url")
+        if not nl:
+            click.echo("\n]")
+    else:
+        response = utils.get(url, token)
+        click.echo(json.dumps(response.json(), indent=4))
+
+
 def load_token(auth):
     try:
         token = json.load(open(auth))["github_personal_token"]

github_to_sqlite/utils.py

@@ -649,3 +649,12 @@ def fetch_emojis(token=None):
 
 def fetch_image(url):
     return requests.get(url).content
+
+
+def get(url, token=None):
+    headers = make_headers(token)
+    if url.startswith("/"):
+        url = "https://api.github.com{}".format(url)
+    response = requests.get(url, headers=headers)
+    response.raise_for_status()
+    return response

tests/test_get.py

@@ -0,0 +1,93 @@
+from click.testing import CliRunner
+from github_to_sqlite import cli
+import pytest
+import textwrap
+
+
+@pytest.fixture
+def mocked_paginated(requests_mock):
+    requests_mock.get(
+        "https://api.github.com/paginated",
+        json=[{"id": 1, "title": "Item 1"}, {"id": 2, "title": "Item 2"}],
+        headers={"link": '<https://api.github.com/paginated?page=2>; rel="next"'},
+    )
+    requests_mock.get(
+        "https://api.github.com/paginated?page=2",
+        json=[{"id": 3, "title": "Item 3"}, {"id": 4, "title": "Item 4"}],
+        headers={"link": '<https://api.github.com/paginated>; rel="prev"'},
+    )
+
+
+@pytest.mark.parametrize("url", ["https://api.github.com/paginated", "/paginated"])
+def test_get(mocked_paginated, url):
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        result = runner.invoke(cli.cli, ["get", url])
+        assert 0 == result.exit_code
+        expected = textwrap.dedent(
+            """
+        [
+            {
+                "id": 1,
+                "title": "Item 1"
+            },
+            {
+                "id": 2,
+                "title": "Item 2"
+            }
+        ]
+        """
+        ).strip()
+        assert result.output.strip() == expected
+
+
+@pytest.mark.parametrize(
+    "nl,expected",
+    (
+        (
+            False,
+            textwrap.dedent(
+                """
+            [
+                {
+                    "id": 1,
+                    "title": "Item 1"
+                },
+                {
+                    "id": 2,
+                    "title": "Item 2"
+                },
+                {
+                    "id": 3,
+                    "title": "Item 3"
+                },
+                {
+                    "id": 4,
+                    "title": "Item 4"
+                }
+            ]"""
+            ).strip(),
+        ),
+        (
+            True,
+            textwrap.dedent(
+                """
+            {"id": 1, "title": "Item 1"}
+            {"id": 2, "title": "Item 2"}
+            {"id": 3, "title": "Item 3"}
+            {"id": 4, "title": "Item 4"}
+            """
+            ).strip(),
+        ),
+    ),
+)
+def test_get_paginate(mocked_paginated, nl, expected):
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            cli.cli,
+            ["get", "https://api.github.com/paginated", "--paginate"]
+            + (["--nl"] if nl else []),
+        )
+        assert 0 == result.exit_code
+        assert result.output.strip() == expected