job.py 9.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318
  1. #!/usr/bin/env python3
  2. import requests
  3. import os
  4. import time
  5. import argparse
  6. import logging
  7. import shutil
  8. import zipfile
  9. logging.basicConfig(
  10. level=logging.INFO,
  11. format="%(asctime)s [%(levelname)s] %(message)s [%(filename)s:%(lineno)d]",
  12. handlers=[logging.StreamHandler()],
  13. )
  14. # The URL of your Flask server
  15. BASE_URL = os.getenv("BASE_URL") or "http://localhost:5000"
  16. # The secret key for API authentication
  17. SECRET_KEY = os.getenv("SECRET_KEY") or "worldpeace2024"
  18. # The headers for API requests
  19. HEADERS = {"Authorization": f"Bearer {SECRET_KEY}"}
  20. SIGN_TIMEOUT = int(os.getenv("SIGN_TIMEOUT") or "30")
  21. TIMEOUT = float(os.getenv("TIMEOUT") or "900")
  22. def create(task_name, file_path=None):
  23. if file_path is None:
  24. response = requests.post(
  25. f"{BASE_URL}/tasks/{task_name}", timeout=TIMEOUT, headers=HEADERS
  26. )
  27. else:
  28. with open(file_path, "rb") as f:
  29. files = {"file": f}
  30. response = requests.post(
  31. f"{BASE_URL}/tasks/{task_name}",
  32. timeout=TIMEOUT,
  33. headers=HEADERS,
  34. files=files,
  35. )
  36. return get_json(response)
  37. def upload_file(task_id, file_path):
  38. with open(file_path, "rb") as f:
  39. files = {"file": f}
  40. response = requests.post(
  41. f"{BASE_URL}/tasks/{task_id}/files",
  42. timeout=TIMEOUT,
  43. headers=HEADERS,
  44. files=files,
  45. )
  46. return get_json(response)
  47. def get_status(task_id):
  48. response = requests.get(
  49. f"{BASE_URL}/tasks/{task_id}/status", timeout=TIMEOUT, headers=HEADERS
  50. )
  51. return get_json(response)
  52. def download_files(task_id, output_dir, fn=None):
  53. response = requests.get(
  54. f"{BASE_URL}/tasks/{task_id}/files",
  55. timeout=TIMEOUT,
  56. headers=HEADERS,
  57. stream=True,
  58. )
  59. # Check if the request was successful
  60. if fn is None:
  61. fn = f"task_{task_id}_files.zip"
  62. if response.status_code == 200:
  63. # Save the file to the output directory
  64. with open(os.path.join(output_dir, fn), "wb") as f:
  65. for chunk in response.iter_content(chunk_size=1024):
  66. if chunk:
  67. f.write(chunk)
  68. return response.ok
  69. def download_one_file(task_id, file_id, output_dir):
  70. response = requests.get(
  71. f"{BASE_URL}/tasks/{task_id}/files/{file_id}",
  72. timeout=TIMEOUT,
  73. headers=HEADERS,
  74. stream=True,
  75. )
  76. # Check if the request was successful
  77. if response.status_code == 200:
  78. # Save the file to the output directory
  79. with open(os.path.join(output_dir, file_id), "wb") as f:
  80. for chunk in response.iter_content(chunk_size=1024):
  81. if chunk:
  82. f.write(chunk)
  83. return response.ok
  84. def fetch(tag=None):
  85. response = requests.get(
  86. f"{BASE_URL}/tasks/fetch_task" + ("?tag=%s" % tag if tag else ""),
  87. timeout=TIMEOUT,
  88. headers=HEADERS,
  89. )
  90. return get_json(response)
  91. def update_status(task_id, status):
  92. response = requests.patch(
  93. f"{BASE_URL}/tasks/{task_id}/status",
  94. timeout=TIMEOUT,
  95. headers=HEADERS,
  96. json=status,
  97. )
  98. return get_json(response)
  99. def delete_task(task_id):
  100. response = requests.delete(
  101. f"{BASE_URL}/tasks/{task_id}",
  102. timeout=TIMEOUT,
  103. headers=HEADERS,
  104. )
  105. return get_json(response)
  106. def sign(file_path):
  107. res = create("sign", file_path)
  108. if res.ok:
  109. task_id = res.task_id
  110. # Poll the status every second
  111. while True:
  112. status = get_status(task_id)
  113. if status["status"] == "done":
  114. # Download the files
  115. download_files(task_id, "output")
  116. # Delete the task
  117. delete_task(task_id)
  118. break
  119. time.sleep(1)
  120. def sign_one_file(file_path):
  121. logging.info(f"Signing {file_path}")
  122. res = create("sign", file_path)
  123. logging.info(f"Uploaded {file_path}")
  124. task_id = res["id"]
  125. n = 0
  126. while True:
  127. if n >= SIGN_TIMEOUT:
  128. delete_task(task_id)
  129. logging.error(f"Failed to sign {file_path}")
  130. break
  131. time.sleep(6)
  132. n += 1
  133. status = get_status(task_id)
  134. if status and status.get("state") == "done":
  135. download_one_file(
  136. task_id, os.path.basename(file_path), os.path.dirname(file_path)
  137. )
  138. delete_task(task_id)
  139. logging.info(f"Signed {file_path}")
  140. return True
  141. return False
  142. def get_json(response):
  143. try:
  144. return response.json()
  145. except Exception as e:
  146. raise Exception(response.text)
  147. SIGN_EXTENSIONS = [
  148. ".dll",
  149. ".exe",
  150. ".sys",
  151. ".vxd",
  152. ".msix",
  153. ".msixbundle",
  154. ".appx",
  155. ".appxbundle",
  156. ".msi",
  157. ".msp",
  158. ".msm",
  159. ".cab",
  160. ".ps1",
  161. ".psm1",
  162. ]
  163. def sign_files(dir_path, only_ext=None):
  164. if only_ext:
  165. only_ext = only_ext.split(",")
  166. for i in range(len(only_ext)):
  167. if not only_ext[i].startswith("."):
  168. only_ext[i] = "." + only_ext[i]
  169. for root, dirs, files in os.walk(dir_path):
  170. for file in files:
  171. file_path = os.path.join(root, file)
  172. _, ext = os.path.splitext(file_path)
  173. if only_ext and ext not in only_ext:
  174. continue
  175. if ext in SIGN_EXTENSIONS:
  176. if not sign_one_file(file_path):
  177. logging.error(f"Failed to sign {file_path}")
  178. break
  179. def main():
  180. parser = argparse.ArgumentParser(
  181. description="Command line interface for task operations."
  182. )
  183. subparsers = parser.add_subparsers(dest="command")
  184. # Create a parser for the "sign_one_file" command
  185. sign_one_file_parser = subparsers.add_parser(
  186. "sign_one_file", help="Sign a single file."
  187. )
  188. sign_one_file_parser.add_argument("file_path", help="The path of the file to sign.")
  189. # Create a parser for the "sign_files" command
  190. sign_files_parser = subparsers.add_parser(
  191. "sign_files", help="Sign all files in a directory."
  192. )
  193. sign_files_parser.add_argument(
  194. "dir_path", help="The path of the directory containing the files to sign."
  195. )
  196. sign_files_parser.add_argument(
  197. "only_ext", help="The file extension to sign.", default=None, nargs="?"
  198. )
  199. # Create a parser for the "fetch" command
  200. fetch_parser = subparsers.add_parser("fetch", help="Fetch a task.")
  201. # Create a parser for the "update_status" command
  202. update_status_parser = subparsers.add_parser(
  203. "update_status", help="Update the status of a task."
  204. )
  205. update_status_parser.add_argument("task_id", help="The ID of the task to update.")
  206. update_status_parser.add_argument("status", help="The new status of the task.")
  207. # Create a parser for the "delete_task" command
  208. delete_task_parser = subparsers.add_parser("delete_task", help="Delete a task.")
  209. delete_task_parser.add_argument("task_id", help="The ID of the task to delete.")
  210. # Create a parser for the "create" command
  211. create_parser = subparsers.add_parser("create", help="Create a task.")
  212. create_parser.add_argument("task_name", help="The name of the task to create.")
  213. create_parser.add_argument(
  214. "file_path",
  215. help="The path of the file for the task.",
  216. default=None,
  217. nargs="?",
  218. )
  219. # Create a parser for the "upload_file" command
  220. upload_file_parser = subparsers.add_parser(
  221. "upload_file", help="Upload a file to a task."
  222. )
  223. upload_file_parser.add_argument(
  224. "task_id", help="The ID of the task to upload the file to."
  225. )
  226. upload_file_parser.add_argument("file_path", help="The path of the file to upload.")
  227. # Create a parser for the "get_status" command
  228. get_status_parser = subparsers.add_parser(
  229. "get_status", help="Get the status of a task."
  230. )
  231. get_status_parser.add_argument(
  232. "task_id", help="The ID of the task to get the status of."
  233. )
  234. # Create a parser for the "download_files" command
  235. download_files_parser = subparsers.add_parser(
  236. "download_files", help="Download files from a task."
  237. )
  238. download_files_parser.add_argument(
  239. "task_id", help="The ID of the task to download files from."
  240. )
  241. download_files_parser.add_argument(
  242. "output_dir", help="The directory to save the downloaded files to."
  243. )
  244. args = parser.parse_args()
  245. if args.command == "sign_one_file":
  246. sign_one_file(args.file_path)
  247. elif args.command == "sign_files":
  248. sign_files(args.dir_path, args.only_ext)
  249. elif args.command == "fetch":
  250. print(fetch())
  251. elif args.command == "update_status":
  252. print(update_status(args.task_id, args.status))
  253. elif args.command == "delete_task":
  254. print(delete_task(args.task_id))
  255. elif args.command == "create":
  256. print(create(args.task_name, args.file_path))
  257. elif args.command == "upload_file":
  258. print(upload_file(args.task_id, args.file_path))
  259. elif args.command == "get_status":
  260. print(get_status(args.task_id))
  261. elif args.command == "download_files":
  262. print(download_files(args.task_id, args.output_dir))
  263. if __name__ == "__main__":
  264. main()