Init

Author: silentEAG

.gitignore

@@ -0,0 +1,16 @@
+# python generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# venv
+.venv
+
+# pytest
+.pytest_cache/
+
+# docs
+docs/

.python-version

@@ -0,0 +1 @@
+3.10.14

README.md

@@ -0,0 +1,6 @@
+# Black Fish
+
+多配置源同步文章爬虫
+
+
+嘶吼、火线 还有 qianxin攻防社区的

pyproject.toml

@@ -0,0 +1,36 @@
+[project]
+name = "black-fish"
+version = "0.1.0"
+description = "Add your description here"
+dependencies = [
+    "drissionpage>=4.0",
+    "asyncio>=3.4.3",
+    "aiohttp>=3.9.5",
+    "loguru>=0.7.2",
+    "parsel>=1.9.1",
+    "markdownify>=0.13.1",
+    "aiofiles>=24.1.0",
+]
+readme = "README.md"
+requires-python = ">= 3.8"
+
+
+[project.scripts]
+dev = 'black_fish:main'
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.rye]
+managed = true
+dev-dependencies = [
+    "pytest>=8.3.1",
+]
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/black_fish"]
+

requirements-dev.lock

@@ -0,0 +1,103 @@
+# generated by rye
+# use `rye lock` or `rye sync` to update this lockfile
+#
+# last locked with the following flags:
+#   pre: false
+#   features: []
+#   all-features: false
+#   with-sources: false
+#   generate-hashes: false
+#   universal: false
+
+-e file:.
+aiofiles==24.1.0
+    # via black-fish
+aiohttp==3.9.5
+    # via black-fish
+aiosignal==1.3.1
+    # via aiohttp
+async-timeout==4.0.3
+    # via aiohttp
+asyncio==3.4.3
+    # via black-fish
+attrs==23.2.0
+    # via aiohttp
+beautifulsoup4==4.12.3
+    # via markdownify
+certifi==2024.7.4
+    # via requests
+charset-normalizer==3.3.2
+    # via requests
+click==8.1.7
+    # via drissionpage
+cssselect==1.2.0
+    # via drissionpage
+    # via parsel
+datarecorder==3.5.1
+    # via downloadkit
+downloadkit==2.0.2
+    # via drissionpage
+drissionpage==4.0.5.6
+    # via black-fish
+et-xmlfile==1.1.0
+    # via openpyxl
+exceptiongroup==1.2.2
+    # via pytest
+filelock==3.15.4
+    # via tldextract
+frozenlist==1.4.1
+    # via aiohttp
+    # via aiosignal
+idna==3.7
+    # via requests
+    # via tldextract
+    # via yarl
+iniconfig==2.0.0
+    # via pytest
+jmespath==1.0.1
+    # via parsel
+loguru==0.7.2
+    # via black-fish
+lxml==5.2.2
+    # via drissionpage
+    # via parsel
+markdownify==0.13.1
+    # via black-fish
+multidict==6.0.5
+    # via aiohttp
+    # via yarl
+openpyxl==3.1.5
+    # via datarecorder
+packaging==24.1
+    # via parsel
+    # via pytest
+parsel==1.9.1
+    # via black-fish
+pluggy==1.5.0
+    # via pytest
+psutil==6.0.0
+    # via drissionpage
+pytest==8.3.1
+requests==2.32.3
+    # via downloadkit
+    # via drissionpage
+    # via requests-file
+    # via tldextract
+requests-file==2.1.0
+    # via tldextract
+six==1.16.0
+    # via markdownify
+soupsieve==2.5
+    # via beautifulsoup4
+tldextract==5.1.2
+    # via drissionpage
+tomli==2.0.1
+    # via pytest
+urllib3==2.2.2
+    # via requests
+w3lib==2.2.1
+    # via parsel
+websocket-client==1.8.0
+    # via drissionpage
+yarl==1.9.4
+    # via aiohttp

requirements.lock

@@ -0,0 +1,93 @@
+# generated by rye
+# use `rye lock` or `rye sync` to update this lockfile
+#
+# last locked with the following flags:
+#   pre: false
+#   features: []
+#   all-features: false
+#   with-sources: false
+#   generate-hashes: false
+#   universal: false
+
+-e file:.
+aiofiles==24.1.0
+    # via black-fish
+aiohttp==3.9.5
+    # via black-fish
+aiosignal==1.3.1
+    # via aiohttp
+async-timeout==4.0.3
+    # via aiohttp
+asyncio==3.4.3
+    # via black-fish
+attrs==23.2.0
+    # via aiohttp
+beautifulsoup4==4.12.3
+    # via markdownify
+certifi==2024.7.4
+    # via requests
+charset-normalizer==3.3.2
+    # via requests
+click==8.1.7
+    # via drissionpage
+cssselect==1.2.0
+    # via drissionpage
+    # via parsel
+datarecorder==3.5.1
+    # via downloadkit
+downloadkit==2.0.2
+    # via drissionpage
+drissionpage==4.0.5.6
+    # via black-fish
+et-xmlfile==1.1.0
+    # via openpyxl
+filelock==3.15.4
+    # via tldextract
+frozenlist==1.4.1
+    # via aiohttp
+    # via aiosignal
+idna==3.7
+    # via requests
+    # via tldextract
+    # via yarl
+jmespath==1.0.1
+    # via parsel
+loguru==0.7.2
+    # via black-fish
+lxml==5.2.2
+    # via drissionpage
+    # via parsel
+markdownify==0.13.1
+    # via black-fish
+multidict==6.0.5
+    # via aiohttp
+    # via yarl
+openpyxl==3.1.5
+    # via datarecorder
+packaging==24.1
+    # via parsel
+parsel==1.9.1
+    # via black-fish
+psutil==6.0.0
+    # via drissionpage
+requests==2.32.3
+    # via downloadkit
+    # via drissionpage
+    # via requests-file
+    # via tldextract
+requests-file==2.1.0
+    # via tldextract
+six==1.16.0
+    # via markdownify
+soupsieve==2.5
+    # via beautifulsoup4
+tldextract==5.1.2
+    # via drissionpage
+urllib3==2.2.2
+    # via requests
+w3lib==2.2.1
+    # via parsel
+websocket-client==1.8.0
+    # via drissionpage
+yarl==1.9.4
+    # via aiohttp

src/black_fish/__init__.py

@@ -0,0 +1,40 @@
+import asyncio
+import sys
+
+from loguru import logger
+
+from black_fish.executor import Executor
+from .config import Config
+from .spiders.xz import XZSpider
+from .spiders.tttang import TTTangSpider
+from .base_spider import BaseArticleSpider
+
+import time
+
+async def runtime():
+    config = Config()
+    executor = Executor(config)
+    xz_spider = XZSpider()
+    # tttang_spider = TTTangSpider()
+    executor.spiders = [xz_spider]
+
+    start = time.time()
+    await executor.run()
+
+    # await tttang_spider.fetch_remote_preview_articles()
+
+    # await xz_spider.prepare_for_run()
+    # await xz_spider.close()
+
+    # f = open("./xz-sample.html", "r")
+    # xz_sample_content = f.read()
+    # xz_spider.parse_to_md(xz_sample_content)
+
+    end = time.time()
+    print(end - start)
+    
+
+def main():
+    logger.remove()
+    logger.add(sys.stdout, colorize=True, format="<green>{time}</green> <level>{message}</level>")
+    asyncio.run(runtime())