Replacing httpbin.org with httpbin.local backed by Docker Compose.

.github/workflows/main.yml

@@ -20,21 +20,6 @@ jobs:
       matrix:
         python-version: ['3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', 'pypy3.9']
 
-    services:
-      # https://docs.github.com/en/actions/using-containerized-services/about-service-containers
-      redis:
-        # Docker Hub image
-        image: redis
-        # Set health checks to wait until redis has started
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps port 6379 on service container to the host
-          - 6379:6379
-
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
@@ -45,9 +30,20 @@ jobs:
         cache-dependency-path: |
           Pipfile
           setup.py
+    - uses: isbang/compose-action@v1.4.1
+      with:
+        compose-file: "./docker-compose.yml"
+        down-flags: "--remove-orphans"
+        up-flags: "--no-start"
     - name: Install dependencies
       run: |
         make develop
+        make services-up
+    - name: Setup hostname
+      run: |
+        export CONTAINER_ID=$(docker-compose ps -q proxy)
+        export CONTAINER_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $CONTAINER_ID)
+        echo "$CONTAINER_IP httpbin.local" | sudo tee -a /etc/hosts
     - name: Test
       run: |
         make test

Makefile

@@ -8,9 +8,15 @@ install-test-requirements:
 	pipenv install --dev
 	pipenv run python -c "import pipfile; pf = pipfile.load('Pipfile'); print('\n'.join(package+version for package, version in pf.data['default'].items()))" > requirements.txt
 
+services-up:
+	pipenv run docker-compose up -d
+
+services-down:
+	pipenv run docker-compose down --remove-orphans
+
 test-python:
 	@echo "Running Python tests"
-	pipenv run python run_tests.py || exit 1
+	pipenv run wait-for-it --service httpbin.local:443 --service localhost:6379 --timeout 5 -- pipenv run python run_tests.py || exit 1
 	@echo ""
 
 lint-python:
@@ -35,6 +41,7 @@ publish: install-test-requirements
 
 clean:
 	rm -rf *.egg-info dist/
-	find . -type d -name __pycache__ -exec rm -rf {} \;
+	find . -type d -name __pycache__ | xargs rf -rf
 
-.PHONY: clean publish safetest test setup develop lint-python test-python install-test-requirements install-dev-requirements
+.PHONY: clean publish safetest test setup develop lint-python test-python
+.PHONY: services-up services-down install-test-requirements install-dev-requirements

Pipfile

@@ -30,6 +30,8 @@ wheel = "*"
 twine = "*"
 anaconda-client = "*"
 fastapi = "*"
+docker-compose = "*"
+wait-for-it = "*"
 
 [requires]
 python_version = "3"

docker-compose.yml

@@ -0,0 +1,15 @@
+services:
+  proxy:
+    container_name: proxy
+    image: shroomlife/docker-https-proxy:latest
+    hostname: "httpbin.local"
+    ports:
+      - "80:80"
+      - "443:443"
+  httpbin:
+    container_name: httpbin.local.proxy
+    image: kennethreitz/httpbin:latest
+  redis:
+    image: redis:latest
+    ports:
+      - "6379:6379"

mocket/mocket.py

@@ -164,7 +164,7 @@ class MocketSocket:
     _secure_socket = False
 
     def __init__(
-        self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, *args, **kwargs
+        self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, **kwargs
     ):
         self.true_socket = true_socket(family, type, proto)
         self._buflen = 65536

tests/main/test_http.py

@@ -30,15 +30,15 @@ def assertEqualHeaders(self, first, second, msg=None):
 class TrueHttpEntryTestCase(HttpTestCase):
     @mocketize
     def test_truesendall(self):
-        url = "http://httpbin.org/ip"
+        url = "http://httpbin.local/ip"
         resp = urlopen(url)
         self.assertEqual(resp.code, 200)
         resp = requests.get(url)
         self.assertEqual(resp.status_code, 200)
 
     @mocketize(truesocket_recording_dir=recording_directory)
     def test_truesendall_with_recording(self):
-        url = "http://httpbin.org/ip"
+        url = "http://httpbin.local/ip"
 
         urlopen(url)
         requests.get(url)
@@ -54,11 +54,11 @@ def test_truesendall_with_recording(self):
         with io.open(dump_filename) as f:
             responses = json.load(f)
 
-        self.assertEqual(len(responses["httpbin.org"]["80"].keys()), 2)
+        self.assertEqual(len(responses["httpbin.local"]["80"].keys()), 2)
 
     @mocketize(truesocket_recording_dir=recording_directory)
     def test_truesendall_with_gzip_recording(self):
-        url = "http://httpbin.org/gzip"
+        url = "http://httpbin.local/gzip"
 
         requests.get(url)
         resp = requests.get(url)
@@ -70,11 +70,11 @@ def test_truesendall_with_gzip_recording(self):
         with io.open(dump_filename) as f:
             responses = json.load(f)
 
-        assert len(responses["httpbin.org"]["80"].keys()) == 1
+        assert len(responses["httpbin.local"]["80"].keys()) == 1
 
     @mocketize(truesocket_recording_dir=recording_directory)
     def test_truesendall_with_chunk_recording(self):
-        url = "http://httpbin.org/range/70000?chunk_size=65536"
+        url = "http://httpbin.local/range/70000?chunk_size=65536"
 
         requests.get(url)
         resp = requests.get(url)
@@ -86,12 +86,14 @@ def test_truesendall_with_chunk_recording(self):
         with io.open(dump_filename) as f:
             responses = json.load(f)
 
-        assert len(responses["httpbin.org"]["80"].keys()) == 1
+        assert len(responses["httpbin.local"]["80"].keys()) == 1
 
     @mocketize
     def test_wrongpath_truesendall(self):
-        Entry.register(Entry.GET, "http://httpbin.org/user.agent", Response(status=404))
-        response = urlopen("http://httpbin.org/ip")
+        Entry.register(
+            Entry.GET, "http://httpbin.local/user.agent", Response(status=404)
+        )
+        response = urlopen("http://httpbin.local/ip")
         self.assertEqual(response.code, 200)
 
 
@@ -205,7 +207,7 @@ def test_mockhttp_entry_collect_duplicates(self):
 
     @mocketize
     def test_multipart(self):
-        url = "http://httpbin.org/post"
+        url = "http://httpbin.local/post"
         data = '--xXXxXXyYYzzz\r\nContent-Disposition: form-data; name="content"\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Length: 68\r\n\r\nAction: comment\nText: Comment with attach\nAttachment: x1.txt, x2.txt\r\n--xXXxXXyYYzzz\r\nContent-Disposition: form-data; name="attachment_2"; filename="x.txt"\r\nContent-Type: text/plain\r\nContent-Length: 4\r\n\r\nbye\n\r\n--xXXxXXyYYzzz\r\nContent-Disposition: form-data; name="attachment_1"; filename="x.txt"\r\nContent-Type: text/plain\r\nContent-Length: 4\r\n\r\nbye\n\r\n--xXXxXXyYYzzz--\r\n'
         headers = {
             "Content-Length": "495",
@@ -229,7 +231,7 @@ def test_multipart(self):
                 "accept-encoding": "identity",
                 "content-length": "495",
                 "content-type": "multipart/form-data; boundary=xXXxXXyYYzzz",
-                "host": "httpbin.org",
+                "host": "httpbin.local",
                 "user-agent": "Mocket",
                 "connection": "keep-alive",
             },
@@ -300,9 +302,11 @@ def test_request_bodies(self):
 
     @mocketize(truesocket_recording_dir=os.path.dirname(__file__))
     def test_truesendall_with_dump_from_recording(self):
-        requests.get("http://httpbin.org/ip", headers={"user-agent": "Fake-User-Agent"})
         requests.get(
-            "http://httpbin.org/gzip", headers={"user-agent": "Fake-User-Agent"}
+            "http://httpbin.local/ip", headers={"user-agent": "Fake-User-Agent"}
+        )
+        requests.get(
+            "http://httpbin.local/gzip", headers={"user-agent": "Fake-User-Agent"}
         )
 
         dump_filename = os.path.join(
@@ -311,7 +315,7 @@ def test_truesendall_with_dump_from_recording(self):
         with io.open(dump_filename) as f:
             responses = json.load(f)
 
-        self.assertEqual(len(responses["httpbin.org"]["80"].keys()), 2)
+        self.assertEqual(len(responses["httpbin.local"]["80"].keys()), 2)
 
     @mocketize
     def test_post_file_object(self):

tests/main/test_http_aiohttp.py

@@ -16,7 +16,7 @@
     class AioHttpEntryTestCase(TestCase):
         @mocketize
         def test_http_session(self):
-            url = "http://httpbin.org/ip"
+            url = "http://httpbin.local/ip"
             body = "asd" * 100
             Entry.single_register(Entry.GET, url, body=body, status=404)
             Entry.single_register(Entry.POST, url, body=body * 2, status=201)
@@ -42,7 +42,7 @@ async def main(_loop):
 
         @mocketize
         def test_https_session(self):
-            url = "https://httpbin.org/ip"
+            url = "https://httpbin.local/ip"
             body = "asd" * 100
             Entry.single_register(Entry.GET, url, body=body, status=404)
             Entry.single_register(Entry.POST, url, body=body * 2, status=201)
@@ -66,7 +66,7 @@ async def main(_loop):
 
         @httprettified
         def test_httprettish_session(self):
-            url = "https://httpbin.org/ip"
+            url = "https://httpbin.local/ip"
             HTTPretty.register_uri(
                 HTTPretty.GET,
                 url,

tests/main/test_http_with_xxhash.py

@@ -6,23 +6,26 @@
 import os
 
 import requests
-from tests.main.test_http import HttpTestCase
 
 from mocket import Mocket, mocketize
+from tests.main.test_http import HttpTestCase
 
 
 class HttpEntryTestCase(HttpTestCase):
-
     @mocketize(truesocket_recording_dir=os.path.dirname(__file__))
     def test_truesendall_with_dump_from_recording(self):
-        requests.get('http://httpbin.org/ip', headers={"user-agent": "Fake-User-Agent"})
-        requests.get('http://httpbin.org/gzip', headers={"user-agent": "Fake-User-Agent"})
+        requests.get(
+            "http://httpbin.local/ip", headers={"user-agent": "Fake-User-Agent"}
+        )
+        requests.get(
+            "http://httpbin.local/gzip", headers={"user-agent": "Fake-User-Agent"}
+        )
 
         dump_filename = os.path.join(
             Mocket.get_truesocket_recording_dir(),
-            Mocket.get_namespace() + '.json',
+            Mocket.get_namespace() + ".json",
         )
         with io.open(dump_filename) as f:
             responses = json.load(f)
 
-        self.assertEqual(len(responses['httpbin.org']['80'].keys()), 2)
+        self.assertEqual(len(responses["httpbin.local"]["80"].keys()), 2)

tests/main/test_httpretty.py

@@ -54,7 +54,10 @@ def test_httpretty_provides_easy_access_to_querystrings():
 
     requests.get("http://yipit.com/?foo=bar&foo=baz&chuck=norris")
     expect(HTTPretty.last_request.querystring).to.equal(
-        {"foo": ["bar", "baz"], "chuck": ["norris"],}
+        {
+            "foo": ["bar", "baz"],
+            "chuck": ["norris"],
+        }
     )
 
 
@@ -118,13 +121,19 @@ def test_httpretty_should_allow_forcing_headers_requests():
         HTTPretty.GET,
         "http://github.com/foo",
         body="<root><baz /</root>",
-        forcing_headers={"Content-Type": "application/xml", "Content-Length": "19",},
+        forcing_headers={
+            "Content-Type": "application/xml",
+            "Content-Length": "19",
+        },
     )
 
     response = requests.get("http://github.com/foo")
 
     expect(dict(response.headers)).to.equal(
-        {"content-type": "application/xml", "content-length": "19",}
+        {
+            "content-type": "application/xml",
+            "content-length": "19",
+        }
     )
 
 
@@ -195,12 +204,18 @@ def test_can_inspect_last_request():
     response = requests.post(
         "http://api.github.com",
         '{"username": "gabrielfalcao"}',
-        headers={"content-type": "text/json",},
+        headers={
+            "content-type": "text/json",
+        },
     )
 
     expect(HTTPretty.last_request.method).to.equal("POST")
-    expect(HTTPretty.last_request.body).to.equal(b'{"username": "gabrielfalcao"}',)
-    expect(HTTPretty.last_request.headers["content-type"]).to.equal("text/json",)
+    expect(HTTPretty.last_request.body).to.equal(
+        b'{"username": "gabrielfalcao"}',
+    )
+    expect(HTTPretty.last_request.headers["content-type"]).to.equal(
+        "text/json",
+    )
     expect(response.json()).to.equal({"repositories": ["HTTPretty", "lettuce"]})
 
 
@@ -216,12 +231,18 @@ def test_can_inspect_last_request_with_ssl():
     response = requests.post(
         "https://secure.github.com",
         '{"username": "gabrielfalcao"}',
-        headers={"content-type": "text/json",},
+        headers={
+            "content-type": "text/json",
+        },
     )
 
     expect(HTTPretty.last_request.method).to.equal("POST")
-    expect(HTTPretty.last_request.body).to.equal(b'{"username": "gabrielfalcao"}',)
-    expect(HTTPretty.last_request.headers["content-type"]).to.equal("text/json",)
+    expect(HTTPretty.last_request.body).to.equal(
+        b'{"username": "gabrielfalcao"}',
+    )
+    expect(HTTPretty.last_request.headers["content-type"]).to.equal(
+        "text/json",
+    )
     expect(response.json()).to.equal({"repositories": ["HTTPretty", "lettuce"]})
 
 
@@ -240,14 +261,15 @@ def test_httpretty_ignores_querystrings_from_registered_uri():
 
 @httprettified
 def test_multiline():
-    url = "http://httpbin.org/post"
+    url = "http://httpbin.local/post"
     data = b"content=Im\r\na multiline\r\n\r\nsentence\r\n"
     headers = {
         "Content-Type": "application/x-www-form-urlencoded; charset=utf-8",
         "Accept": "text/plain",
     }
     HTTPretty.register_uri(
-        HTTPretty.POST, url,
+        HTTPretty.POST,
+        url,
     )
     response = requests.post(url, data=data, headers=headers)
 
@@ -264,15 +286,16 @@ def test_multiline():
 
 @httprettified
 def test_multipart():
-    url = "http://httpbin.org/post"
+    url = "http://httpbin.local/post"
     data = b'--xXXxXXyYYzzz\r\nContent-Disposition: form-data; name="content"\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Length: 68\r\n\r\nAction: comment\nText: Comment with attach\nAttachment: x1.txt, x2.txt\r\n--xXXxXXyYYzzz\r\nContent-Disposition: form-data; name="attachment_2"; filename="x.txt"\r\nContent-Type: text/plain\r\nContent-Length: 4\r\n\r\nbye\n\r\n--xXXxXXyYYzzz\r\nContent-Disposition: form-data; name="attachment_1"; filename="x.txt"\r\nContent-Type: text/plain\r\nContent-Length: 4\r\n\r\nbye\n\r\n--xXXxXXyYYzzz--\r\n'
     headers = {
         "Content-Length": "495",
         "Content-Type": "multipart/form-data; boundary=xXXxXXyYYzzz",
         "Accept": "text/plain",
     }
     HTTPretty.register_uri(
-        HTTPretty.POST, url,
+        HTTPretty.POST,
+        url,
     )
     response = requests.post(url, data=data, headers=headers)
     expect(response.status_code).to.equal(200)