diff --git a/Pipfile.lock b/Pipfile.lock index 9dcc036..c073f07 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -50,11 +50,11 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:594ca51a10d2b3443cbac41214e12dbb2a1cd57e1a7344659849e2e20ba6a8d8", - "sha256:a4bbe77fd30670455c5296242967a123ec28c37e9702a8a81bd2f20a4baf0368", - "sha256:d4e96ac9b0c3a6d3f0caae2e4124e6055c5dcafde8e2f831ff194c104f0775a0" + "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7", + "sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8", + "sha256:e718f2342e2e099b640a34ab782407b7b676f47ee272d6739e60b8ea23829f2c" ], - "version": "==4.9.0" + "version": "==4.9.1" }, "bs4": { "hashes": [ @@ -79,11 +79,11 @@ }, "cloudscraper": { "hashes": [ - "sha256:06eb4fd7462dc08a193228830f45097993efc8af4fd75a74815ba16a05c6a0fd", - "sha256:dec9d92a323e85d390af8d02e475de425604212bc6e50c78c0897bf05d355352" + "sha256:3893be3c281ddb7a39bf35e558da19247a39a408ef48078fdabf5058b7659d6c", + "sha256:db295c5ca33f22ae058f317b07c6842a2b16d75c9e11e38d21395363d089692f" ], "index": "pypi", - "version": "==1.2.36" + "version": "==1.2.38" }, "colorama": { "hashes": [ @@ -143,33 +143,33 @@ }, "loguru": { "hashes": [ - "sha256:074b3caa6748452c1e4f2b302093c94b65d5a4c5a4d7743636b4121e06437b0e", - "sha256:a6101fd435ac89ba5205a105a26a6ede9e4ddbb4408a6e167852efca47806d11" + "sha256:1e0e6ff59be5e22f863d909ca989e34bb14c21b374f6af45281e603d003dbb96", + "sha256:4688d9e1f31d70e1ec7ccce5305967bc28f377eb1048d009108c11faebe05bcf" ], "index": "pypi", - "version": "==0.4.1" + "version": "==0.5.0" }, "multidict": { "hashes": [ - "sha256:317f96bc0950d249e96d8d29ab556d01dd38888fbe68324f46fd834b430169f1", - "sha256:42f56542166040b4474c0c608ed051732033cd821126493cf25b6c276df7dd35", - "sha256:4b7df040fb5fe826d689204f9b544af469593fb3ff3a069a6ad3409f742f5928", - "sha256:544fae9261232a97102e27a926019100a9db75bec7b37feedd74b3aa82f29969", - "sha256:620b37c3fea181dab09267cd5a84b0f23fa043beb8bc50d8474dd9694de1fa6e", - "sha256:6e6fef114741c4d7ca46da8449038ec8b1e880bbe68674c01ceeb1ac8a648e78", - "sha256:7774e9f6c9af3f12f296131453f7b81dabb7ebdb948483362f5afcaac8a826f1", - "sha256:85cb26c38c96f76b7ff38b86c9d560dea10cf3459bb5f4caf72fc1bb932c7136", - "sha256:a326f4240123a2ac66bb163eeba99578e9d63a8654a59f4688a79198f9aa10f8", - "sha256:ae402f43604e3b2bc41e8ea8b8526c7fa7139ed76b0d64fc48e28125925275b2", - "sha256:aee283c49601fa4c13adc64c09c978838a7e812f85377ae130a24d7198c0331e", - "sha256:b51249fdd2923739cd3efc95a3d6c363b67bbf779208e9f37fd5e68540d1a4d4", - "sha256:bb519becc46275c594410c6c28a8a0adc66fe24fef154a9addea54c1adb006f5", - "sha256:c2c37185fb0af79d5c117b8d2764f4321eeb12ba8c141a95d0aa8c2c1d0a11dd", - "sha256:dc561313279f9d05a3d0ffa89cd15ae477528ea37aa9795c4654588a3287a9ab", - "sha256:e439c9a10a95cb32abd708bb8be83b2134fa93790a4fb0535ca36db3dda94d20", - "sha256:fc3b4adc2ee8474cb3cd2a155305d5f8eda0a9c91320f83e55748e1fcb68f8e3" + "sha256:1ece5a3369835c20ed57adadc663400b5525904e53bae59ec854a5d36b39b21a", + "sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000", + "sha256:3750f2205b800aac4bb03b5ae48025a64e474d2c6cc79547988ba1d4122a09e2", + "sha256:4538273208e7294b2659b1602490f4ed3ab1c8cf9dbdd817e0e9db8e64be2507", + "sha256:5141c13374e6b25fe6bf092052ab55c0c03d21bd66c94a0e3ae371d3e4d865a5", + "sha256:51a4d210404ac61d32dada00a50ea7ba412e6ea945bbe992e4d7a595276d2ec7", + "sha256:5cf311a0f5ef80fe73e4f4c0f0998ec08f954a6ec72b746f3c179e37de1d210d", + "sha256:6513728873f4326999429a8b00fc7ceddb2509b01d5fd3f3be7881a257b8d463", + "sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19", + "sha256:9456e90649005ad40558f4cf51dbb842e32807df75146c6d940b6f5abb4a78f3", + "sha256:c026fe9a05130e44157b98fea3ab12969e5b60691a276150db9eda71710cd10b", + "sha256:d14842362ed4cf63751648e7672f7174c9818459d169231d03c56e84daf90b7c", + "sha256:e0d072ae0f2a179c375f67e3da300b47e1a83293c554450b29c900e50afaae87", + "sha256:f07acae137b71af3bb548bd8da720956a3bc9f9a0b87733e0899226a2317aeb7", + "sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430", + "sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255", + "sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d" ], - "version": "==4.7.5" + "version": "==4.7.6" }, "openpyxl": { "hashes": [ @@ -242,50 +242,59 @@ }, "six": { "hashes": [ - "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", - "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "version": "==1.14.0" + "version": "==1.15.0" }, "soupsieve": { "hashes": [ - "sha256:e914534802d7ffd233242b785229d5ba0766a7f487385e3f714446a07bf540ae", - "sha256:fcd71e08c0aee99aca1b73f45478549ee7e7fc006d51b37bec9e9def7dc22b69" + "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55", + "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232" ], - "version": "==2.0" + "version": "==2.0.1" }, "sqlalchemy": { "hashes": [ - "sha256:083e383a1dca8384d0ea6378bd182d83c600ed4ff4ec8247d3b2442cf70db1ad", - "sha256:0a690a6486658d03cc6a73536d46e796b6570ac1f8a7ec133f9e28c448b69828", - "sha256:114b6ace30001f056e944cebd46daef38fdb41ebb98f5e5940241a03ed6cad43", - "sha256:128f6179325f7597a46403dde0bf148478f868df44841348dfc8d158e00db1f9", - "sha256:13d48cd8b925b6893a4e59b2dfb3e59a5204fd8c98289aad353af78bd214db49", - "sha256:211a1ce7e825f7142121144bac76f53ac28b12172716a710f4bf3eab477e730b", - "sha256:2dc57ee80b76813759cccd1a7affedf9c4dbe5b065a91fb6092c9d8151d66078", - "sha256:3e625e283eecc15aee5b1ef77203bfb542563fa4a9aa622c7643c7b55438ff49", - "sha256:43078c7ec0457387c79b8d52fff90a7ad352ca4c7aa841c366238c3e2cf52fdf", - "sha256:5b1bf3c2c2dca738235ce08079783ef04f1a7fc5b21cf24adaae77f2da4e73c3", - "sha256:6056b671aeda3fc451382e52ab8a753c0d5f66ef2a5ccc8fa5ba7abd20988b4d", - "sha256:68d78cf4a9dfade2e6cf57c4be19f7b82ed66e67dacf93b32bb390c9bed12749", - "sha256:7025c639ce7e170db845e94006cf5f404e243e6fc00d6c86fa19e8ad8d411880", - "sha256:7224e126c00b8178dfd227bc337ba5e754b197a3867d33b9f30dc0208f773d70", - "sha256:7d98e0785c4cd7ae30b4a451416db71f5724a1839025544b4edbd92e00b91f0f", - "sha256:8d8c21e9d4efef01351bf28513648ceb988031be4159745a7ad1b3e28c8ff68a", - "sha256:bbb545da054e6297242a1bb1ba88e7a8ffb679f518258d66798ec712b82e4e07", - "sha256:d00b393f05dbd4ecd65c989b7f5a81110eae4baea7a6a4cdd94c20a908d1456e", - "sha256:e18752cecaef61031252ca72031d4d6247b3212ebb84748fc5d1a0d2029c23ea" + "sha256:128bc917ed20d78143a45024455ff0aed7d3b96772eba13d5dbaf9cc57e5c41b", + "sha256:156a27548ba4e1fed944ff9fcdc150633e61d350d673ae7baaf6c25c04ac1f71", + "sha256:27e2efc8f77661c9af2681755974205e7462f1ae126f498f4fe12a8b24761d15", + "sha256:2a12f8be25b9ea3d1d5b165202181f2b7da4b3395289000284e5bb86154ce87c", + "sha256:31c043d5211aa0e0773821fcc318eb5cbe2ec916dfbc4c6eea0c5188971988eb", + "sha256:65eb3b03229f684af0cf0ad3bcc771970c1260a82a791a8d07bffb63d8c95bcc", + "sha256:6cd157ce74a911325e164441ff2d9b4e244659a25b3146310518d83202f15f7a", + "sha256:703c002277f0fbc3c04d0ae4989a174753a7554b2963c584ce2ec0cddcf2bc53", + "sha256:869bbb637de58ab0a912b7f20e9192132f9fbc47fc6b5111cd1e0f6cdf5cf9b0", + "sha256:8a0e0cd21da047ea10267c37caf12add400a92f0620c8bc09e4a6531a765d6d7", + "sha256:8d01e949a5d22e5c4800d59b50617c56125fc187fbeb8fa423e99858546de616", + "sha256:925b4fe5e7c03ed76912b75a9a41dfd682d59c0be43bce88d3b27f7f5ba028fb", + "sha256:9cb1819008f0225a7c066cac8bb0cf90847b2c4a6eb9ebb7431dbd00c56c06c5", + "sha256:a87d496884f40c94c85a647c385f4fd5887941d2609f71043e2b73f2436d9c65", + "sha256:a9030cd30caf848a13a192c5e45367e3c6f363726569a56e75dc1151ee26d859", + "sha256:a9e75e49a0f1583eee0ce93270232b8e7bb4b1edc89cc70b07600d525aef4f43", + "sha256:b50f45d0e82b4562f59f0e0ca511f65e412f2a97d790eea5f60e34e5f1aabc9a", + "sha256:b7878e59ec31f12d54b3797689402ee3b5cfcb5598f2ebf26491732758751908", + "sha256:ce1ddaadee913543ff0154021d31b134551f63428065168e756d90bdc4c686f5", + "sha256:ce2646e4c0807f3461be0653502bb48c6e91a5171d6e450367082c79e12868bf", + "sha256:ce6c3d18b2a8ce364013d47b9cad71db815df31d55918403f8db7d890c9d07ae", + "sha256:e4e2664232005bd306f878b0f167a31f944a07c4de0152c444f8c61bbe3cfb38", + "sha256:e8aa395482728de8bdcca9cc0faf3765ab483e81e01923aaa736b42f0294f570", + "sha256:eb4fcf7105bf071c71068c6eee47499ab8d4b8f5a11fc35147c934f0faa60f23", + "sha256:ed375a79f06cad285166e5be74745df1ed6845c5624aafadec4b7a29c25866ef", + "sha256:f35248f7e0d63b234a109dd72fbfb4b5cb6cb6840b221d0df0ecbf54ab087654", + "sha256:f502ef245c492b391e0e23e94cba030ab91722dcc56963c85bfd7f3441ea2bbe", + "sha256:fe01bac7226499aedf472c62fa3b85b2c619365f3f14dd222ffe4f3aa91e5f98" ], "markers": "python_version >= '3.0'", - "version": "==1.3.16" + "version": "==1.3.17" }, "tablib": { "hashes": [ - "sha256:4d1909aa3ff1c85ba97ad16176c0aeec33c8e894dc7ea6f10f2dd44701e99ba7", - "sha256:80f6c3453431cedf1125f23d16b3d96b92b426495714ebf0b4dede1fa75b447d" + "sha256:8cc2fa10bc37219ac5e76850eb7fbd50de313c7a1a7895c44af2a8dd206b7be7", + "sha256:c5a77c96ad306d5b380def1309d521ad5e98b4f83726f84857ad87e142d13279" ], "index": "pypi", - "version": "==1.1.0" + "version": "==2.0.0" }, "tenacity": { "hashes": [ diff --git a/config/log.py b/config/log.py index 4f3ebfc..20af969 100644 --- a/config/log.py +++ b/config/log.py @@ -23,12 +23,12 @@ logfile_fmt = '{time:YYYY-MM-DD HH:mm:ss,SSS} ' \ '{line} - {message}' logger.remove() -logger.level(name='TRACE', no=5, color='', icon='✏️') -logger.level(name='DEBUG', no=10, color='', icon='🐞 ') +logger.level(name='TRACE', color='', icon='✏️') +logger.level(name='DEBUG', color='', icon='🐞 ') logger.level(name='INFOR', no=20, color='', icon='ℹ️') logger.level(name='QUITE', no=25, color='', icon='🤫 ') logger.level(name='ALERT', no=30, color='', icon='⚠️') -logger.level(name='ERROR', no=40, color='', icon='❌️') +logger.level(name='ERROR', color='', icon='❌️') logger.level(name='FATAL', no=50, color='', icon='☠️') # 如果你想在命令终端静默运行OneForAll,可以将以下一行中的level设置为QUITE diff --git a/modules/collect.py b/modules/collect.py index 8fdfd73..c8ae637 100644 --- a/modules/collect.py +++ b/modules/collect.py @@ -31,7 +31,7 @@ class Collect(object): # The crawl module has some problems modules = ['certificates', 'check', 'datasets', 'dnsquery', 'intelligence', 'search'] - # modules = ['intelligence'] # The crawl module has some problems + # modules = ['datasets'] for module in modules: module_path = setting.module_dir.joinpath(module) for path in module_path.rglob('*.py'): diff --git a/requirements.txt b/requirements.txt index cbce5af..1136b87 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ aiohttp==3.6.2 async-timeout==3.0.1 attrs==19.3.0 -beautifulsoup4==4.9.0 +beautifulsoup4==4.9.1 bs4==0.0.1 certifi==2020.4.5.1 chardet==3.0.4 -cloudscraper==1.2.36 +cloudscraper==1.2.38 colorama==0.4.3 dnspython==1.16.0 docopt==0.6.2 @@ -14,8 +14,8 @@ exrex==0.10.5 fire==0.3.1 idna==2.9 jdcal==1.4.1 -loguru==0.4.1 -multidict==4.7.5 +loguru==0.5.0 +multidict==4.7.6 openpyxl==2.4.11 psutil==5.7.0 pyparsing==2.4.7 @@ -24,10 +24,10 @@ records==0.5.3 requests==2.23.0 requests-file==1.5.1 requests-toolbelt==0.9.1 -six==1.14.0 -soupsieve==2.0 -SQLAlchemy==1.3.16 -tablib==1.1.0 +six==1.15.0 +soupsieve==2.0.1 +SQLAlchemy==1.3.17 +tablib==2.0.0 tenacity==6.2.0 termcolor==1.1.0 tldextract==2.2.2