From 6660e129a87bb2aa4fb0b31009488712b1341551 Mon Sep 17 00:00:00 2001 From: NinjaCheetah <58050615+NinjaCheetah@users.noreply.github.com> Date: Tue, 17 Dec 2024 21:55:01 -0500 Subject: [PATCH 1/3] Added WIP JSON-based script support, removed old format The old script code was already broken from the previous commits anyway, since they changed too much internally. This new format is much easier to understand and will allow for creating more powerful scripts. --- NUSGet.py | 130 ++++++++++++++++++-------------------- modules/core.py | 9 +++ modules/download_batch.py | 36 +++++++++++ 3 files changed, 105 insertions(+), 70 deletions(-) create mode 100644 modules/download_batch.py diff --git a/NUSGet.py b/NUSGet.py index 26101d6..c19a90f 100644 --- a/NUSGet.py +++ b/NUSGet.py @@ -34,8 +34,9 @@ from qt.py.ui_MainMenu import Ui_MainWindow from modules.core import * from modules.tree import NUSGetTreeModel -from modules.download_wii import run_nus_download_wii, run_nus_download_wii_batch -from modules.download_dsi import run_nus_download_dsi, run_nus_download_dsi_batch +from modules.download_batch import run_nus_download_batch +from modules.download_wii import run_nus_download_wii +from modules.download_dsi import run_nus_download_dsi nusget_version = "1.3.0" @@ -333,86 +334,75 @@ class MainWindow(QMainWindow, Ui_MainWindow): msg_box.setStandardButtons(QMessageBox.StandardButton.Ok) msg_box.setDefaultButton(QMessageBox.StandardButton.Ok) msg_box.setWindowTitle(app.translate("MainWindow", "Script Download Failed")) - file_name = QFileDialog.getOpenFileName(self, caption=app.translate("MainWindow", "Open NUS script"), - filter=app.translate("MainWindow", "NUS Scripts (*.nus *.txt)"), + file_name = QFileDialog.getOpenFileName(self, caption=app.translate("MainWindow", "Open NUS Script"), + filter=app.translate("MainWindow", "NUS Scripts (*.nus *.json)"), options=QFileDialog.Option.ReadOnly) + # The old plaintext script format is no longer supported in NUSGet v1.3.0 and later. This script parsing code + # is for the new JSON script format, which is much easier to use and is cleaner. if len(file_name[0]) == 0: return try: - content = open(file_name[0], "r").readlines() - except os.error: - msg_box.setText(app.translate("MainWindow", "The script could not be opened.")) + with open(file_name[0]) as script_file: + script_data = json.load(script_file) + except json.JSONDecodeError as e: + msg_box.setText(app.translate("MainWindow", "An error occurred while parsing the script file!")) + msg_box.setInformativeText(app.translate("MainWindow", f"Error encountered at line {e.lineno}, column {e.colno}. Please double-check the script and try again.")) msg_box.exec() return - - # NUS Scripts are plaintext UTF-8 files that list a title per line, terminated with newlines. - # Every title is its u64 TID, a space and its u16 version, *both* written in hexadecimal. - # NUS itself expects versions as decimal notation, so they need to be decoded first, but TIDs are always written - # in hexadecimal notation. + # Build a list of the titles we need to download. titles = [] - for index, title in enumerate(content): - decoded = title.replace("\n", "").split(" ", 1) - if len(decoded[0]) != 16: - msg_box.setText(app.translate("MainWindow", "The TID for title #%n is not valid.", "", index + 1)) - msg_box.exec() - return - elif len(decoded[1]) != 4: - msg_box.setText(app.translate("MainWindow", "The version for title #%n is not valid.", "", index + 1)) - msg_box.exec() - return - - tid = decoded[0] - + for title in script_data: try: - target_version = int(decoded[1], 16) - except ValueError: - msg_box.setText(app.translate("MainWindow", "The version for title #%n is not valid.", "", index + 1)) + tid = title["Title ID"] + except KeyError: + msg_box.setText(app.translate("MainWindow", "An error occurred while parsing Title IDs!")) + msg_box.setInformativeText(app.translate("MainWindow", f"The title at index {script_data.index(title)} does not have a Title ID!")) msg_box.exec() return - - title = None - for category in self.trees[self.ui.platform_tabs.currentIndex()][1]: - for title_ in self.trees[self.ui.platform_tabs.currentIndex()][1][category]: - # The last two digits are either identifying the title type (IOS slot, BC type, etc.) or a region code; in case of the latter, skip the region here to match it - if not ((title_["TID"][-2:] == "XX" and title_["TID"][:-2] == tid[:-2]) or title_["TID"] == tid): - continue - - found_ver = False - for region in title_["Versions"]: - for db_version in title_["Versions"][region]: - if db_version == target_version: - found_ver = True + # No version key is acceptable, just treat it as latest. + try: + title_version = int(title["Version"]) + except KeyError: + title_version = -1 + # If no console was specified, assume Wii. + try: + console = title["Console"] + except KeyError: + console = "Wii" + # Look up the title, and load the archive name for it if one can be found. + archive_name = "" + if console == "vWii": + target_database = vwii_database + elif console == "DSi": + target_database = dsi_database + else: + target_database = wii_database + for category in target_database: + for t in target_database[category]: + if t["TID"][-2:] == "XX": + for r in regions: + if f"{t['TID'][:-2]}{regions[r][0]}" == tid: + try: + archive_name = t["Archive Name"] + break + except KeyError: + archive_name = "" + break + else: + if t["TID"] == tid: + try: + archive_name = t["Archive Name"] break - - if not found_ver: - msg_box.setText(app.translate("MainWindow", "The version for title #%n could not be discovered in the database.", "", index + 1)) - msg_box.exec() - return - - title = title_ - break - - if title is None: - msg_box.setText(app.translate("MainWindow", "Title #%n could not be discovered in the database.", "", index + 1)) - msg_box.exec() - return - - titles.append((title["TID"], str(target_version), title["Archive Name"])) - + except KeyError: + archive_name = "" + break + print(archive_name) + titles.append(BatchTitleData(tid, title_version, console, archive_name)) self.lock_ui_for_download() - - self.update_log_text(f"Found {len(titles)} titles, starting batch download.") - - if self.ui.console_select_dropdown.currentText() == "DSi": - worker = Worker(run_nus_download_dsi_batch, out_folder, titles, self.ui.pack_archive_chkbox.isChecked(), - self.ui.keep_enc_chkbox.isChecked(), self.ui.create_dec_chkbox.isChecked(), - self.ui.use_local_chkbox.isChecked(), self.ui.archive_file_entry.text()) - else: - worker = Worker(run_nus_download_wii_batch, out_folder, titles, self.ui.pack_archive_chkbox.isChecked(), - self.ui.keep_enc_chkbox.isChecked(), self.ui.create_dec_chkbox.isChecked(), - self.ui.use_wiiu_nus_chkbox.isChecked(), self.ui.use_local_chkbox.isChecked(), - self.ui.pack_vwii_mode_chkbox.isChecked(), self.ui.patch_ios_chkbox.isChecked()) - + worker = Worker(run_nus_download_batch, out_folder, titles, self.ui.pack_archive_chkbox.isChecked(), + self.ui.keep_enc_chkbox.isChecked(), self.ui.create_dec_chkbox.isChecked(), + self.ui.use_wiiu_nus_chkbox.isChecked(), self.ui.use_local_chkbox.isChecked(), + self.ui.pack_vwii_mode_chkbox.isChecked(), self.ui.patch_ios_chkbox.isChecked()) worker.signals.result.connect(self.check_download_result) worker.signals.progress.connect(self.update_log_text) self.threadpool.start(worker) diff --git a/modules/core.py b/modules/core.py index 8eba0bb..e0123ce 100644 --- a/modules/core.py +++ b/modules/core.py @@ -18,6 +18,15 @@ class TitleData: danger: str +@dataclass +class BatchTitleData: + # Class to store all data for a Title in a batch operation. + tid: str + version: int + console: str + archive_name: str + + def check_nusget_updates(app, current_version: str, progress_callback=None) -> str | None: # Simple function to make a request to the GitHub API and then check if the latest available version is newer. gh_api_request = requests.get(url="https://api.github.com/repos/NinjaCheetah/NUSGet/releases/latest", stream=True) diff --git a/modules/download_batch.py b/modules/download_batch.py new file mode 100644 index 0000000..d01e383 --- /dev/null +++ b/modules/download_batch.py @@ -0,0 +1,36 @@ +# "modules/download_batch.py", licensed under the MIT license +# Copyright 2024 NinjaCheetah + +import pathlib +from typing import List +from modules.core import BatchTitleData +from modules.download_dsi import run_nus_download_dsi +from modules.download_wii import run_nus_download_wii + + +def run_nus_download_batch(out_folder: pathlib.Path, titles: List[BatchTitleData], pack_wad_chkbox: bool, + keep_enc_chkbox: bool, decrypt_contents_chkbox: bool, wiiu_nus_chkbox: bool, + use_local_chkbox: bool, repack_vwii_chkbox: bool, patch_ios: bool, + progress_callback=None): + for title in titles: + if title.console == "Wii" or title.console == "vWii": + if title.archive_name != "": + archive_name = f"{title.archive_name}-v{title.version}-{title.console}.wad" + else: + archive_name = f"{title.tid}-v{title.version}-{title.console}.wad" + result = run_nus_download_wii(out_folder, title.tid, str(title.version), pack_wad_chkbox, keep_enc_chkbox, + decrypt_contents_chkbox, wiiu_nus_chkbox, use_local_chkbox, repack_vwii_chkbox, + patch_ios, archive_name, progress_callback) + if result != 0: + return result + elif title.console == "DSi": + if title.archive_name != "": + archive_name = f"{title.archive_name}-v{title.version}-{title.console}.tad" + else: + archive_name = f"{title.tid}-v{title.version}-{title.console}.tad" + result = run_nus_download_dsi(out_folder, title.tid, str(title.version), pack_wad_chkbox, keep_enc_chkbox, + decrypt_contents_chkbox, use_local_chkbox, archive_name, progress_callback) + if result != 0: + return result + progress_callback.emit(f"Batch download finished.") + return 0 From 87da32d62eb428049ae710789791d8c4a0ea73ed Mon Sep 17 00:00:00 2001 From: NinjaCheetah <58050615+NinjaCheetah@users.noreply.github.com> Date: Wed, 18 Dec 2024 09:08:21 -0500 Subject: [PATCH 2/3] Improved WAD name formatting for batch downloads Also started working on a fix for the checkmark/X icons denoting a ticket not appearing anymore. --- NUSGet.py | 1 - data/wii-database.json | 12 ++++++++++++ modules/download_batch.py | 16 ++++++++++------ modules/download_dsi.py | 14 ++------------ modules/download_wii.py | 15 ++------------- modules/tree.py | 12 +++++++----- 6 files changed, 33 insertions(+), 37 deletions(-) diff --git a/NUSGet.py b/NUSGet.py index c19a90f..ba24bea 100644 --- a/NUSGet.py +++ b/NUSGet.py @@ -396,7 +396,6 @@ class MainWindow(QMainWindow, Ui_MainWindow): except KeyError: archive_name = "" break - print(archive_name) titles.append(BatchTitleData(tid, title_version, console, archive_name)) self.lock_ui_for_download() worker = Worker(run_nus_download_batch, out_folder, titles, self.ui.pack_archive_chkbox.isChecked(), diff --git a/data/wii-database.json b/data/wii-database.json index 94cfce4..52ca14e 100644 --- a/data/wii-database.json +++ b/data/wii-database.json @@ -859,5 +859,17 @@ }, "Ticket": false } + ], + "Virtual Console - NES": [ + { + "Name": "Super Mario Bros.", + "TID": "00010001464147XX", + "Versions": { + "USA/NTSC": [2], + "Europe/PAL": [2], + "Japan": [2] + }, + "Ticket": false + } ] } \ No newline at end of file diff --git a/modules/download_batch.py b/modules/download_batch.py index d01e383..e2083be 100644 --- a/modules/download_batch.py +++ b/modules/download_batch.py @@ -13,22 +13,26 @@ def run_nus_download_batch(out_folder: pathlib.Path, titles: List[BatchTitleData use_local_chkbox: bool, repack_vwii_chkbox: bool, patch_ios: bool, progress_callback=None): for title in titles: + if title.version == -1: + version_str = "Latest" + else: + version_str = str(title.version) if title.console == "Wii" or title.console == "vWii": if title.archive_name != "": - archive_name = f"{title.archive_name}-v{title.version}-{title.console}.wad" + archive_name = f"{title.archive_name}-v{version_str}-{title.console}.wad" else: - archive_name = f"{title.tid}-v{title.version}-{title.console}.wad" - result = run_nus_download_wii(out_folder, title.tid, str(title.version), pack_wad_chkbox, keep_enc_chkbox, + archive_name = f"{title.tid}-v{version_str}-{title.console}.wad" + result = run_nus_download_wii(out_folder, title.tid, version_str, pack_wad_chkbox, keep_enc_chkbox, decrypt_contents_chkbox, wiiu_nus_chkbox, use_local_chkbox, repack_vwii_chkbox, patch_ios, archive_name, progress_callback) if result != 0: return result elif title.console == "DSi": if title.archive_name != "": - archive_name = f"{title.archive_name}-v{title.version}-{title.console}.tad" + archive_name = f"{title.archive_name}-v{version_str}-{title.console}.tad" else: - archive_name = f"{title.tid}-v{title.version}-{title.console}.tad" - result = run_nus_download_dsi(out_folder, title.tid, str(title.version), pack_wad_chkbox, keep_enc_chkbox, + archive_name = f"{title.tid}-v{version_str}-{title.console}.tad" + result = run_nus_download_dsi(out_folder, title.tid, version_str, pack_wad_chkbox, keep_enc_chkbox, decrypt_contents_chkbox, use_local_chkbox, archive_name, progress_callback) if result != 0: return result diff --git a/modules/download_dsi.py b/modules/download_dsi.py index 47f7e86..285acc4 100644 --- a/modules/download_dsi.py +++ b/modules/download_dsi.py @@ -99,6 +99,8 @@ def run_nus_download_dsi(out_folder: pathlib.Path, tid: str, version: str, pack_ # Use a typed TAD name if there is one, and auto generate one based on the TID and version if there isn't. progress_callback.emit("Packing TAD...") if tad_file_name != "" and tad_file_name is not None: + # Batch downloads may insert -vLatest, so if it did we can fill in the real number now. + tad_file_name = tad_file_name.replace("-vLatest", f"-v{title_version}") if tad_file_name[-4:].lower() != ".tad": tad_file_name += ".tad" else: @@ -112,15 +114,3 @@ def run_nus_download_dsi(out_folder: pathlib.Path, tid: str, version: str, pack_ if (not pack_tad_enabled and pack_tad_chkbox) or (not decrypt_contents_enabled and decrypt_contents_chkbox): return 1 return 0 - -def run_nus_download_dsi_batch(out_folder: pathlib.Path, titles: List[Tuple[str, str, str]], pack_tad_chkbox: bool, - keep_enc_chkbox: bool, decrypt_contents_chkbox: bool, use_local_chkbox: bool, - progress_callback=None): - for title in titles: - result = run_nus_download_dsi(out_folder, title[0], title[1], pack_tad_chkbox, keep_enc_chkbox, - decrypt_contents_chkbox, use_local_chkbox, f"{title[2]}-{title[1]}.tad", - progress_callback) - if result != 0: - return result - progress_callback.emit(f"Batch download finished.") - return 0 diff --git a/modules/download_wii.py b/modules/download_wii.py index 0e436b4..5ee485c 100644 --- a/modules/download_wii.py +++ b/modules/download_wii.py @@ -116,6 +116,8 @@ def run_nus_download_wii(out_folder: pathlib.Path, tid: str, version: str, pack_ # Use a typed WAD name if there is one, and auto generate one based on the TID and version if there isn't. progress_callback.emit(" - Packing WAD...") if wad_file_name != "" and wad_file_name is not None: + # Batch downloads may insert -vLatest, so if it did we can fill in the real number now. + wad_file_name = wad_file_name.replace("-vLatest", f"-v{title_version}") if wad_file_name[-4:].lower() != ".wad": wad_file_name += ".wad" else: @@ -140,16 +142,3 @@ def run_nus_download_wii(out_folder: pathlib.Path, tid: str, version: str, pack_ if (not pack_wad_enabled and pack_wad_chkbox) or (not decrypt_contents_enabled and decrypt_contents_chkbox): return 1 return 0 - -def run_nus_download_wii_batch(out_folder: pathlib.Path, titles: List[Tuple[str, str, str]], pack_wad_chkbox: bool, - keep_enc_chkbox: bool, decrypt_contents_chkbox: bool, wiiu_nus_chkbox: bool, - use_local_chkbox: bool, repack_vwii_chkbox: bool, patch_ios: bool, - progress_callback=None): - for title in titles: - result = run_nus_download_wii(out_folder, title[0], title[1], pack_wad_chkbox, keep_enc_chkbox, - decrypt_contents_chkbox, wiiu_nus_chkbox, use_local_chkbox, repack_vwii_chkbox, - patch_ios, f"{title[2]}-{title[1]}.wad", progress_callback) - if result != 0: - return result - progress_callback.emit(f"Batch download finished.") - return 0 diff --git a/modules/tree.py b/modules/tree.py index d3df9fd..1df28c0 100644 --- a/modules/tree.py +++ b/modules/tree.py @@ -60,7 +60,9 @@ class NUSGetTreeModel(QAbstractItemModel): tid_item.add_child(region_item) for version in version_list: danger = entry.get("Danger") if entry.get("Danger") is not None else "" - metadata = TitleData(entry.get("TID"), entry.get("Name"), entry.get("Archive Name"), + archive_name = (entry.get("Archive Name") if entry.get("Archive Name") is not None + else entry.get("Name").replace(" ", "-")) + metadata = TitleData(entry.get("TID"), entry.get("Name"), archive_name, version, entry.get("Ticket"), region, key, danger) public_versions = entry.get("Public Versions") if public_versions is not None: @@ -97,11 +99,11 @@ class NUSGetTreeModel(QAbstractItemModel): if role == Qt.DecorationRole and index.column() == 0: # Check for icons based on the "Ticket" metadata only at the TID level if item.parent and item.parent.data_at(0) == "System": - if item.metadata and "Ticket" in item.metadata: - if item.metadata["Ticket"]: - return QIcon.fromTheme("dialog-ok") # Checkmark icon + if item.metadata and item.metadata.ticket: + if item.metadata.ticket: + return QIcon.fromTheme("dialog-ok") else: - return QIcon.fromTheme("dialog-cancel") # X icon + return QIcon.fromTheme("dialog-cancel") return None def headerData(self, section, orientation, role=Qt.DisplayRole): From 31f47f2acd9d239348f4780fbfc2b8c162680e7f Mon Sep 17 00:00:00 2001 From: NinjaCheetah <58050615+NinjaCheetah@users.noreply.github.com> Date: Wed, 18 Dec 2024 16:38:14 -0500 Subject: [PATCH 3/3] Fixed checkmarks/Xs for showing if a ticket is available --- modules/tree.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/modules/tree.py b/modules/tree.py index 1df28c0..87dd2b4 100644 --- a/modules/tree.py +++ b/modules/tree.py @@ -11,7 +11,7 @@ class TreeItem: self.data = data self.parent = parent self.children = [] - self.metadata = metadata # Store hidden metadata + self.metadata = metadata def add_child(self, item): self.children.append(item) @@ -53,7 +53,7 @@ class NUSGetTreeModel(QAbstractItemModel): name = entry.get("Name") versions = entry.get("Versions", {}) if tid: - tid_item = TreeItem([f"{tid} - {name}", ""], key_item) + tid_item = TreeItem([f"{tid} - {name}", ""], key_item, entry.get("Ticket")) key_item.add_child(tid_item) for region, version_list in versions.items(): region_item = TreeItem([region, ""], tid_item) @@ -98,12 +98,11 @@ class NUSGetTreeModel(QAbstractItemModel): if role == Qt.DecorationRole and index.column() == 0: # Check for icons based on the "Ticket" metadata only at the TID level - if item.parent and item.parent.data_at(0) == "System": - if item.metadata and item.metadata.ticket: - if item.metadata.ticket: - return QIcon.fromTheme("dialog-ok") - else: - return QIcon.fromTheme("dialog-cancel") + if item.metadata is not None and isinstance(item.metadata, bool): + if item.metadata is True: + return QIcon.fromTheme("dialog-ok") + else: + return QIcon.fromTheme("dialog-cancel") return None def headerData(self, section, orientation, role=Qt.DisplayRole):