feat: save raw data of lc problems

Author: yanglbme

.gitignore

@@ -1,8 +1,10 @@
-.idea/
-.DS_Store
-.vscode
-/node_modules
-/solution/result.json
-/lcof/lcof.json
-/lcof/lcof_list.json
-/lcci/lcci.json
+.idea/
+.DS_Store
+.vscode
+/node_modules
+/solution/result.json
+/solution/raw.json
+/lcof/lcof.json
+/lcof/lcof_list.json
+/lcci/lcci.json
+/solution/__pycache__

solution/main.py

@@ -1,6 +1,7 @@
 import json
 import os
-from urllib.parse import quote
+import re
+from urllib.parse import quote, unquote
 
 from spider import Spider
 
@@ -130,16 +131,77 @@ def generate_summary(result):
         f.write(summary_en)
 
 
+def refresh(result):
+    """update problems"""
+    pattern = re.compile("src=\"(.*?)\"")
+
+    for question in result:
+        front_question_id = question['frontend_question_id']
+        print(front_question_id)
+
+        path_cn = unquote(str(question['relative_path_cn']).replace("/solution", "."))
+        path_en = unquote(str(question['relative_path_en']).replace("/solution", "."))
+
+        with open(path_cn, 'r', encoding='utf-8') as f1:
+            cn_content = f1.read()
+
+        with open(path_en, 'r', encoding='utf-8') as f2:
+            en_content = f2.read()
+
+        # update question content
+        old_content = re.search("<!-- 这里写题目描述 -->(.*?)## 解法", cn_content, re.S).group(1)
+        cn_content = cn_content.replace(
+            old_content, "\n\n" + question['content_cn'] + "\n\n"
+        ).replace("\n\n    <ul>", "\n    <ul>")
+
+        # replace image url to cdn link
+        for url in pattern.findall(cn_content) or []:
+            image_name = (
+                os.path.basename(url).replace('.PNG', '.png').replace('.JPG', '.jpg')
+            )
+            new_url = (
+                'https://cdn.jsdelivr.net/gh/doocs/leetcode@main'
+                + str(question['relative_path_cn']).replace("README.md", "images/")
+                + image_name
+            )
+            cn_content = cn_content.replace(url, new_url)
+
+        with open(path_cn, 'w', encoding='utf-8') as f1:
+            f1.write(cn_content)
+
+        old_content = re.search(
+            "## Description(.*?)## Solutions", en_content, re.S
+        ).group(1)
+        en_content = en_content.replace(
+            old_content, "\n\n" + question['content_en'] + "\n\n"
+        ).replace("\n\n    <ul>", "\n    <ul>")
+
+        for url in pattern.findall(en_content) or []:
+            image_name = (
+                os.path.basename(url).replace('.PNG', '.png').replace('.JPG', '.jpg')
+            )
+            new_url = (
+                'https://cdn.jsdelivr.net/gh/doocs/leetcode@main'
+                + str(question['relative_path_cn']).replace("README.md", "images/")
+                + image_name
+            )
+            en_content = en_content.replace(url, new_url)
+
+        with open(path_en, 'w', encoding='utf-8') as f2:
+            f2.write(en_content)
+
+
 def save(result):
     with open('./result.json', 'w', encoding='utf-8') as f:
         f.write(json.dumps(result))
 
 
 if __name__ == '__main__':
-    cookie_cn = ''
-    cookie_en = ''
+    cookie_cn = '__auc=85ca7fb417508357e2a5a4c54bf; gr_user_id=4a6501f1-6d25-4b24-80a9-00a43b35e715; _ga=GA1.2.656607691.1602160525; grwng_uid=ce9f2e3b-15df-4f37-ac78-9d6e5f9ca018; __atuvc=1|21; csrftoken=pgozvZH6lu5Ls6O1p1tvEAGftxoUhwVM5AnmH64UkQ9KdcmvlBf2atIgAnBjU7lZ; a2873925c34ecbd2_gr_last_sent_cs1=lcbin; p_h5_u=944F20A8-E0F4-47CE-91F4-EC1D36B69DB2; selectedStreamLevel=HD; _bl_uid=qdk87wb3z3bo66uL4ztbke7r0gq4; _gid=GA1.2.955304545.1648025340; aliyungf_tc=a037027675496c4afd16779c436088ceb980404f1221fa66676cdec7e09c0911; Hm_lvt_fa218a3ff7179639febdb15e372f411c=1649898032,1649927306,1649983681,1650021377; __appToken__=; NEW_PROBLEMLIST_PAGE=1; a2873925c34ecbd2_gr_session_id=bc42d2c4-2755-47c1-b509-b763d341260c; a2873925c34ecbd2_gr_last_sent_sid_with_cs1=bc42d2c4-2755-47c1-b509-b763d341260c; a2873925c34ecbd2_gr_session_id_bc42d2c4-2755-47c1-b509-b763d341260c=true; _gat_gtag_UA_131851415_1=1; LEETCODE_SESSION=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJuZXh0X2FmdGVyX29hdXRoIjoiL3Byb2JsZW1zL2xvbmdlc3Qtc3Vic3RyaW5nLW9mLW9uZS1yZXBlYXRpbmctY2hhcmFjdGVyLyIsIl9hdXRoX3VzZXJfaWQiOiIyNTQwMyIsIl9hdXRoX3VzZXJfYmFja2VuZCI6ImRqYW5nby5jb250cmliLmF1dGguYmFja2VuZHMuTW9kZWxCYWNrZW5kIiwiX2F1dGhfdXNlcl9oYXNoIjoiYTkwOWQwNTQzODU2NDgyYzhhNTA3ZDhiYjY4OTdlNTM1Y2EyZGYyOTU2ZTc5MzBiZjMwNzViMGMyZTNmYWIzZiIsImlkIjoyNTQwMywiZW1haWwiOiJjb250YWN0QHlhbmdsaWJpbi5pbmZvIiwidXNlcm5hbWUiOiJsY2JpbiIsInVzZXJfc2x1ZyI6ImxjYmluIiwiYXZhdGFyIjoiaHR0cHM6Ly9hc3NldHMubGVldGNvZGUtY24uY29tL2FsaXl1bi1sYy11cGxvYWQvdXNlcnMvYmluZ29vby9hdmF0YXJfMTUyMjY0OTQ3Ny5wbmciLCJwaG9uZV92ZXJpZmllZCI6dHJ1ZSwiX3RpbWVzdGFtcCI6MTY0ODEwNDg4OC4zMzM1NDE2LCJleHBpcmVkX3RpbWVfIjoxNjUwNjU0MDAwLCJ2ZXJzaW9uX2tleV8iOjAsImxhdGVzdF90aW1lc3RhbXBfIjoxNjUwMDc0NTQyfQ.89QnOykZYcsKG-AH_xa0E0fZkbP7wd_iCdTzO-mFDa4; Hm_lpvt_fa218a3ff7179639febdb15e372f411c=1650074549; a2873925c34ecbd2_gr_cs1=lcbin'
+    cookie_en = 'gr_user_id=726c2dd2-240d-4971-b0f5-37744121657c; 87b5a3c3f1a55520_gr_last_sent_cs1=bingooo; intercom-id-pq9rak4o=aa42fcc8-64e1-40b5-b596-8dd44c90a142; _ga_DKXQ03QCVK=GS1.1.1628508904.1.1.1628508911.53; _ga=GA1.2.1636041299.1607561338; csrftoken=ACzGOlaLR7I4Om6CLZxy3dU0AHb0DhIt5IlxSMDun7wrugCAHGnFweYculeWZdTB; _gid=GA1.2.1318533002.1650004268; 87b5a3c3f1a55520_gr_cs1=bingooo; c_a_u="YmluZ29vbw==:1nfXlr:mPRm5NKTQwfomYC0IW3sqCLF3b8"; LEETCODE_SESSION=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJfYXV0aF91c2VyX2lkIjoiMTE4NzEwNyIsIl9hdXRoX3VzZXJfYmFja2VuZCI6ImFsbGF1dGguYWNjb3VudC5hdXRoX2JhY2tlbmRzLkF1dGhlbnRpY2F0aW9uQmFja2VuZCIsIl9hdXRoX3VzZXJfaGFzaCI6ImEyM2FkMWMyOTBhNDY1MTliNDg2YjMwNjljNDFjMjg3ZjVhYzI4YWUiLCJpZCI6MTE4NzEwNywiZW1haWwiOiJjb250YWN0QHlhbmdsaWJpbi5pbmZvIiwidXNlcm5hbWUiOiJiaW5nb29vIiwidXNlcl9zbHVnIjoiYmluZ29vbyIsImF2YXRhciI6Imh0dHBzOi8vYXNzZXRzLmxlZXRjb2RlLmNvbS91c2Vycy9iaW5nb29vL2F2YXRhcl8xNTIyNjgxMzQwLnBuZyIsInJlZnJlc2hlZF9hdCI6MTY0OTkwMjA1NSwiaXAiOiIxODMuMTQuMzEuNjEiLCJpZGVudGl0eSI6Ijg3MjI1MGZlYmRiM2JjOGY3OWY1YmQxOWI5YTk2YzRhIiwiY29udmVyc2lvbl90YXJnZXRzIjp7IlJ4dGFWeEVLQVFSSEdVVlFTaGdYRlFNVVRCUUdBQndFRFF0QkZGZGJTMThhSFZKUUZoc1hBZz09Ijp7InNlbmRfc2Vzc2lvbl9pZHMiOlsxNTE1XSwiZW1haWwiOiJjb250YWN0QHlhbmdsaWJpbi5pbmZvIn0sIlJ4dGFWeEVLQVFSSEQxQmNEZ01MWFFzWFcwMEFIQVpkV2t3TUZnPT0iOnsic2VuZF9zZXNzaW9uX2lkcyI6WzE1MTVdLCJlbWFpbCI6ImNvbnRhY3RAeWFuZ2xpYmluLmluZm8ifX0sInNlc3Npb25faWQiOjE3Njc3NTE2fQ.3uixtnnmvtoW5AgeuKxoz6Wl3r8iAe05hKsenFTrQW0; _gat=1'
     spider = Spider(cookie_cn, cookie_en)
     res = spider.run()
+    save(res)
 
     # with open('./result.json', 'r', encoding='utf-8') as f:
     #     res = f.read()
@@ -148,4 +210,4 @@ def save(result):
     generate_readme(res)
     generate_question_readme(res)
     generate_summary(res)
-    save(res)
+    # refresh(res)

solution/refresh.py

@@ -90,6 +90,7 @@ def __init__(self, cookie_cn: str, cookie_en: str):
         self.cookie_cn = cookie_cn
         self.cookie_en = cookie_en
         self.session = requests.session()
+        self.raw_data = {}
 
     def get_all_questions(self) -> List:
         """获取所有题目"""
@@ -192,6 +193,7 @@ def handle(self, question: dict) -> dict:
         url_cn = f'https://leetcode-cn.com/problems/{question_title_slug}'
         url_en = f'https://leetcode.com/problems/{question_title_slug}'
         frontend_question_id = str(question['stat']['frontend_question_id']).zfill(4)
+        self.raw_data[frontend_question_id] = question_detail
         no = int(frontend_question_id) // 100
         question_title_en = question['stat']['question__title']
         question_title_en = re.sub(r'[\\/:*?"<>|]', '', question_title_en).strip()
@@ -251,6 +253,10 @@ def handle(self, question: dict) -> dict:
         item['md_table_row_en'] = [col1_en, col2_en, col3_en, col4_en, col5_en]
         return item
 
+    def save(self):
+        with open('./raw.json', 'w', encoding='utf-8') as f:
+            f.write(json.dumps(self.raw_data))
+
     def run(self):
         questions = self.get_all_questions()
         details = [self.handle(question) for question in questions]
@@ -259,4 +265,5 @@ def run(self):
         ]
         details += [self.handle(question) for question in failed_questions]
         details = [detail for detail in details if detail]
+        self.save()
         return details