Revert changes related to processing content indices

Changes released in libWiiPy v0.5.0 and v0.5.1 to how indices were handled ended up way overcomplicating things, resulting in lots of issues now that I'm working with the content module again in WiiPy. These changes have mostly been reverted.
The issues were related to handling WADs where the content indices don't align with the actual index of the content, like in cases where content has bene removed. This issue has been fixed again with a new and much simpler patch that should not introduce new bugs.

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "libWiiPy"
-version = "0.5.1"
+version = "0.5.2"
 authors = [
     { name="NinjaCheetah", email="ninjacheetah@ncxprogramming.com" },
     { name="Lillian Skinner", email="lillian@randommeaninglesscharacters.com" }

src/libWiiPy/title/commonkeys.py

@@ -7,11 +7,14 @@ common_key = 'ebe42a225e8593e448d9c5457381aaf7'
 korean_key = '63b82bb4f4614e2e13f2fefbba4c9b7e'
 vwii_key = '30bfc76e7c19afbb23163330ced7c28d'
 
+development_key = 'a1604a6a7123b529ae8bec32c816fcaa'
 
-def get_common_key(common_key_index) -> bytes:
+
+def get_common_key(common_key_index, dev=False) -> bytes:
     """
     Gets the specified Wii Common Key based on the index provided. If an invalid common key index is provided, this
-    function falls back on always returning key 0 (the Common Key).
+    function falls back on always returning key 0 (the Common Key). If the kwarg "dev" is specified, then key 0 will
+    point to the development common key rather than the retail one. Keys 1 and 2 are unaffected by this argument.
 
     Possible values for common_key_index: 0: Common Key, 1: Korean Key, 2: vWii Key
 
@@ -19,6 +22,8 @@ def get_common_key(common_key_index) -> bytes:
     ----------
     common_key_index : int
         The index of the common key to be returned.
+    dev : bool
+        If the dev keys should be used in place of the retail keys. Only affects key 0.
 
     Returns
     -------
@@ -27,7 +32,10 @@ def get_common_key(common_key_index) -> bytes:
     """
     match common_key_index:
         case 0:
-            common_key_bin = binascii.unhexlify(common_key)
+            if dev:
+                common_key_bin = binascii.unhexlify(development_key)
+            else:
+                common_key_bin = binascii.unhexlify(common_key)
         case 1:
             common_key_bin = binascii.unhexlify(korean_key)
         case 2:

src/libWiiPy/title/content.py

@@ -117,10 +117,6 @@ class ContentRegion:
         """
         Gets an individual content from the content region based on the provided index, in encrypted form.
 
-        This uses the content index, which is the value tied to each content and used as the IV for encryption, rather
-        than the literal index in the array of content, because sometimes the contents end up out of order in a WAD
-        while still retaining the original indices.
-
         Parameters
         ----------
         index : int
@@ -131,17 +127,10 @@ class ContentRegion:
         bytes
             The encrypted content listed in the content record.
         """
-        # Get a list of the current content indices, so we can make sure the target one exists. Doing it this way
+        if index >= self.num_contents:
-        # ensures we can find the target, even if the highest content index is greater than the highest literal index.
+            raise ValueError(f"You are trying to get the content at index {index}, but no content with that "
-        current_indices = []
+                             f"index exists!")
-        for record in self.content_records:
+        content_enc = self.content_list[index]
-            current_indices.append(record.index)
-        if index not in current_indices:
-            raise ValueError("You are trying to get the content at index " + str(index) + ", but no content with that "
-                             "index exists!")
-        # This is the literal index in the list of content that we're going to get.
-        target_index = current_indices.index(index)
-        content_enc = self.content_list[target_index]
         return content_enc
 
     def get_enc_content_by_cid(self, cid: int) -> bytes:
@@ -181,14 +170,10 @@ class ContentRegion:
         """
         Gets an individual content from the content region based on the provided index, in decrypted form.
 
-        This uses the content index, which is the value tied to each content and used as the IV for encryption, rather
-        than the literal index in the array of content, because sometimes the contents end up out of order in a WAD
-        while still retaining the original indices.
-
         Parameters
         ----------
         index : int
-            The content index of the content you want to get.
+            The index of the content you want to get.
         title_key : bytes
             The Title Key for the title the content is from.
         skip_hash : bool, optional
@@ -199,19 +184,14 @@ class ContentRegion:
         bytes
             The decrypted content listed in the content record.
         """
-        # Get a list of the current content indices, so we can make sure the target one exists. Doing it this way
+        # Get the content index in the Content Record to ensure decryption works properly.
-        # ensures we can find the target, even if the highest content index is greater than the highest literal index.
+        cnt_index = self.content_records[index].index
-        current_indices = []
-        for record in self.content_records:
-            current_indices.append(record.index)
-        # This is the literal index in the list of content that we're going to get.
-        target_index = current_indices.index(index)
         content_enc = self.get_enc_content_by_index(index)
-        content_dec = decrypt_content(content_enc, title_key, index, self.content_records[target_index].content_size)
+        content_dec = decrypt_content(content_enc, title_key, cnt_index, self.content_records[index].content_size)
         # Hash the decrypted content and ensure that the hash matches the one in its Content Record.
         # If it does not, then something has gone wrong in the decryption, and an error will be thrown.
         content_dec_hash = hashlib.sha1(content_dec).hexdigest()
-        content_record_hash = str(self.content_records[target_index].content_hash.decode())
+        content_record_hash = str(self.content_records[index].content_hash.decode())
         # Compare the hash and throw a ValueError if the hash doesn't match.
         if content_dec_hash != content_record_hash:
             if skip_hash:
@@ -273,9 +253,7 @@ class ContentRegion:
 
     def get_index_from_cid(self, cid: int) -> int:
         """
-        Gets the content index of a content by its Content ID. The returned index is the value tied to each content and
+        Gets the index of a content by its Content ID.
-        used as the IV for encryption, rather than the literal index in the array of content, because sometimes the
-        contents end up out of order in a WAD while still retaining the original indices.
 
         Parameters
         ----------
@@ -293,9 +271,8 @@ class ContentRegion:
             content_ids.append(record.content_id)
         if cid not in content_ids:
             raise ValueError("The specified Content ID does not exist!")
-        literal_index = content_ids.index(cid)
+        index = content_ids.index(cid)
-        target_index = self.content_records[literal_index].index
+        return index
-        return target_index
 
     def add_enc_content(self, enc_content: bytes, cid: int, index: int, content_type: int, content_size: int,
                         content_hash: bytes) -> None:
@@ -327,6 +304,7 @@ class ContentRegion:
         # If we're good, then append all the data and create a new ContentRecord().
         self.content_list.append(enc_content)
         self.content_records.append(_ContentRecord(cid, index, content_type, content_size, content_hash))
+        self.num_contents += 1
 
     def add_content(self, dec_content: bytes, cid: int, content_type: int, title_key: bytes) -> None:
         """
@@ -363,18 +341,14 @@ class ContentRegion:
         """
         Sets the content at the provided content index to the provided new encrypted content. The provided hash and
         content size are set in the corresponding content record. A new Content ID or content type can also be
-        specified, but if it isn't than the current values are preserved.
+        specified, but if it isn't then the current values are preserved.
-
-        This uses the content index, which is the value tied to each content and used as the IV for encryption, rather
-        than the literal index in the array of content, because sometimes the contents end up out of order in a WAD
-        while still retaining the original indices.
 
         Parameters
         ----------
         enc_content : bytes
             The new encrypted content to set.
         index : int
-            The target content index to set the new content at.
+            The target index to set the new content at.
         content_size : int
             The size of the new encrypted content when decrypted.
         content_hash : bytes
@@ -384,34 +358,27 @@ class ContentRegion:
         content_type : int, optional
             The type of the new content. Current value will be preserved if not set.
         """
-        # Get a list of the current content indices, so we can make sure the target one exists. Doing it this way
+        if index >= self.num_contents:
-        # ensures we can find the target, even if the highest content index is greater than the highest literal index.
+            raise ValueError(f"You are trying to set the content at index {index}, but no content with that "
-        current_indices = []
+                             f"index currently exists!")
-        for record in self.content_records:
-            current_indices.append(record.index)
-        if index not in current_indices:
-            raise ValueError("You are trying to set the content at index " + str(index) + ", but no content with that "
-                             "index currently exists!")
-        # This is the literal index in the list of content/content records that we're going to change.
-        target_index = current_indices.index(index)
         # Reassign the values, but only set the optional ones if they were passed.
-        self.content_records[target_index].content_size = content_size
+        self.content_records[index].content_size = content_size
-        self.content_records[target_index].content_hash = content_hash
+        self.content_records[index].content_hash = content_hash
         if cid is not None:
-            self.content_records[target_index].content_id = cid
+            self.content_records[index].content_id = cid
         if content_type is not None:
-            self.content_records[target_index].content_type = content_type
+            self.content_records[index].content_type = content_type
         # Add blank entries to the list to ensure that its length matches the length of the content record list.
         while len(self.content_list) < len(self.content_records):
             self.content_list.append(b'')
-        self.content_list[target_index] = enc_content
+        self.content_list[index] = enc_content
 
     def set_content(self, dec_content: bytes, index: int, title_key: bytes, cid: int = None,
                     content_type: int = None) -> None:
         """
         Sets the content at the provided content index to the provided new decrypted content. The hash and content size
         of this content will be generated and then set in the corresponding content record. A new Content ID or content
-        type can also be specified, but if it isn't than the current values are preserved.
+        type can also be specified, but if it isn't then the current values are preserved.
 
         The provided Title Key is used to encrypt the content so that it can be set in the ContentRegion.
 
@@ -432,8 +399,9 @@ class ContentRegion:
         content_size = len(dec_content)
         # Calculate the hash of the new content.
         content_hash = str.encode(hashlib.sha1(dec_content).hexdigest())
-        # Encrypt the content using the provided Title Key and index.
+        # Encrypt the content using the provided Title Key and the index from the Content Record, to ensure that
-        enc_content = encrypt_content(dec_content, title_key, index)
+        # encryption will succeed even if the provided index doesn't match the content's index.
+        enc_content = encrypt_content(dec_content, title_key, self.content_records[index].index)
         # Pass values to set_enc_content()
         self.set_enc_content(enc_content, index, content_size, content_hash, cid, content_type)
 
@@ -443,10 +411,6 @@ class ContentRegion:
         it matches the record at that index. Not recommended for most use cases, use decrypted content and
         load_content() instead.
 
-        This uses the content index, which is the value tied to each content and used as the IV for encryption, rather
-        than the literal index in the array of content, because sometimes the contents end up out of order in a WAD
-        while still retaining the original indices.
-
         Parameters
         ----------
         enc_content : bytes
@@ -454,20 +418,13 @@ class ContentRegion:
         index : int
             The content index to load the content at.
         """
-        # Get a list of the current content indices, so we can make sure the target one exists. Doing it this way
+        if index >= self.num_contents:
-        # ensures we can find the target, even if the highest content index is greater than the highest literal index.
+            raise ValueError(f"You are trying to load the content at index {index}, but no content with that "
-        current_indices = []
+                             f"index currently exists! Make sure the correct content records have been loaded.")
-        for record in self.content_records:
-            current_indices.append(record.index)
-        if index not in current_indices:
-            raise ValueError("You are trying to load the content at index " + str(index) + ", but no content with that "
-                             "index currently exists! Make sure the correct content records have been loaded.")
         # Add blank entries to the list to ensure that its length matches the length of the content record list.
         while len(self.content_list) < len(self.content_records):
             self.content_list.append(b'')
-        # This is the literal index in the list of content/content records that we're going to change.
+        self.content_list[index] = enc_content
-        target_index = current_indices.index(index)
-        self.content_list[target_index] = enc_content
 
     def load_content(self, dec_content: bytes, index: int, title_key: bytes) -> None:
         """
@@ -475,32 +432,21 @@ class ContentRegion:
         sure that it matches the corresponding record. This content will then be encrypted using the provided Title Key
         before being loaded.
 
-        This uses the content index, which is the value tied to each content and used as the IV for encryption, rather
-        than the literal index in the array of content, because sometimes the contents end up out of order in a WAD
-        while still retaining the original indices.
-
         Parameters
         ----------
         dec_content : bytes
             The decrypted content to load.
         index : int
-            The content index to load the content at.
+            The index to load the content at.
         title_key: bytes
             The Title Key that matches the decrypted content.
         """
-        # Get a list of the current content indices, so we can make sure the target one exists. Doing it this way
+        if index >= self.num_contents:
-        # ensures we can find the target, even if the highest content index is greater than the highest literal index.
+            raise ValueError(f"You are trying to load the content at index {index}, but no content with that "
-        current_indices = []
+                             f"index currently exists! Make sure the correct content records have been loaded.")
-        for record in self.content_records:
-            current_indices.append(record.index)
-        if index not in current_indices:
-            raise ValueError("You are trying to load the content at index " + str(index) + ", but no content with that "
-                             "index currently exists! Make sure the correct content records have been loaded.")
-        # This is the literal index in the list of content/content records that we're going to change.
-        target_index = current_indices.index(index)
         # Check the hash of the content against the hash stored in the record to ensure it matches.
         content_hash = hashlib.sha1(dec_content).hexdigest()
-        if content_hash != self.content_records[target_index].content_hash.decode():
+        if content_hash != self.content_records[index].content_hash.decode():
             raise ValueError("The decrypted content provided does not match the record at the provided index. \n"
                              "Expected hash is: {}\n".format(self.content_records[index].content_hash.decode()) +
                              "Actual hash is: {}".format(content_hash))
@@ -508,11 +454,10 @@ class ContentRegion:
         while len(self.content_list) < len(self.content_records):
             self.content_list.append(b'')
         # If the hash matches, encrypt the content and set it where it belongs.
-        # This uses the index from the content records instead of just the index given, because there are some strange
+        # This uses the index from the content records instead of just the index given, because there are some poorly
-        # circumstances where the actual index in the array and the assigned content index don't match up, and this
+        # made custom WADs out there that don't have the contents in order, for whatever reason.
-        # needs to accommodate that. Seems to only apply to custom WADs ? (Like cIOS WADs?)
+        enc_content = encrypt_content(dec_content, title_key, self.content_records[index].index)
-        enc_content = encrypt_content(dec_content, title_key, index)
+        self.content_list[index] = enc_content
-        self.content_list[target_index] = enc_content
 
     def remove_content_by_index(self, index: int) -> None:
         """
@@ -525,19 +470,13 @@ class ContentRegion:
         index : int
             The index of the content you want to remove.
         """
-        # Get a list of the current content indices, so we can make sure the target one exists. Doing it this way
+        if index >= self.num_contents:
-        # ensures we can find the target, even if the highest content index is greater than the highest literal index.
+            raise ValueError(f"You are trying to remove the content at index {index}, but no content with "
-        current_indices = []
+                             f"that index currently exists!")
-        for record in self.content_records:
-            current_indices.append(record.index)
-        if index not in current_indices:
-            raise ValueError("You are trying to remove the content at index " + str(index) + ", but no content with "
-                             "that index currently exists!")
-        # This is the literal index in the list of content/content records that we're going to change.
-        target_index = current_indices.index(index)
         # Delete the target index from both the content list and content records.
-        self.content_list.pop(target_index)
+        self.content_list.pop(index)
-        self.content_records.pop(target_index)
+        self.content_records.pop(index)
+        self.num_contents -= 1
 
     def remove_content_by_cid(self, cid: int) -> None:
         """
@@ -551,11 +490,11 @@ class ContentRegion:
             The Content ID of the content you want to remove.
         """
         try:
-            content_index = self.get_index_from_cid(cid)
+            index = self.get_index_from_cid(cid)
         except ValueError:
             raise ValueError(f"You are trying to remove content with Content ID {cid}, "
                              f"but no content with that ID exists!")
-        self.remove_content_by_index(content_index)
+        self.remove_content_by_index(index)
 
 
 @_dataclass

src/libWiiPy/title/crypto.py

@@ -30,7 +30,7 @@ def _convert_tid_to_iv(title_id: str | bytes) -> bytes:
     return title_key_iv
 
 
-def decrypt_title_key(title_key_enc: bytes, common_key_index: int, title_id: bytes | str) -> bytes:
+def decrypt_title_key(title_key_enc: bytes, common_key_index: int, title_id: bytes | str, dev=False) -> bytes:
     """
     Gets the decrypted version of the encrypted Title Key provided.
 
@@ -44,6 +44,8 @@ def decrypt_title_key(title_key_enc: bytes, common_key_index: int, title_id: byt
         The index of the common key used to encrypt the Title Key.
     title_id : bytes, str
         The Title ID of the title that the key is for.
+    dev : bool
+        Whether the Title Key is encrypted with the development key or not.
 
     Returns
     -------
@@ -51,7 +53,7 @@ def decrypt_title_key(title_key_enc: bytes, common_key_index: int, title_id: byt
         The decrypted Title Key.
     """
     # Load the correct common key for the title.
-    common_key = get_common_key(common_key_index)
+    common_key = get_common_key(common_key_index, dev)
     # Convert the IV into the correct format based on the type provided.
     title_key_iv = _convert_tid_to_iv(title_id)
     # The IV will always be in the same format by this point, so add the last 8 bytes.
@@ -63,7 +65,7 @@ def decrypt_title_key(title_key_enc: bytes, common_key_index: int, title_id: byt
     return title_key
 
 
-def encrypt_title_key(title_key_dec: bytes, common_key_index: int, title_id: bytes | str) -> bytes:
+def encrypt_title_key(title_key_dec: bytes, common_key_index: int, title_id: bytes | str, dev=False) -> bytes:
     """
     Encrypts the provided Title Key with the selected common key.
 
@@ -77,6 +79,8 @@ def encrypt_title_key(title_key_dec: bytes, common_key_index: int, title_id: byt
         The index of the common key used to encrypt the Title Key.
     title_id : bytes, str
         The Title ID of the title that the key is for.
+    dev : bool
+        Whether the Title Key is encrypted with the development key or not.
 
     Returns
     -------
@@ -84,7 +88,7 @@ def encrypt_title_key(title_key_dec: bytes, common_key_index: int, title_id: byt
         An encrypted Title Key.
     """
     # Load the correct common key for the title.
-    common_key = get_common_key(common_key_index)
+    common_key = get_common_key(common_key_index, dev)
     # Convert the IV into the correct format based on the type provided.
     title_key_iv = _convert_tid_to_iv(title_id)
     # The IV will always be in the same format by this point, so add the last 8 bytes.

src/libWiiPy/title/ticket.py

@@ -40,6 +40,9 @@ class Ticket:
 
     Attributes
     ----------
+    is_dev : bool
+        Whether this Ticket is signed for development or not, and whether the Title Key is encrypted for development
+        or not.
     signature : bytes
         The signature applied to the ticket.
     ticket_version : int
@@ -56,6 +59,8 @@ class Ticket:
         The index of the common key required to decrypt this ticket's Title Key.
     """
     def __init__(self):
+        # If this is a dev ticket
+        self.is_dev: bool = False  # Defaults to false, set to true during load if this ticket is using dev certs.
         # Signature blob header
         self.signature_type: bytes = b''  # Type of signature, always 0x10001 for RSA-2048
         self.signature: bytes = b''  # Actual signature data
@@ -155,6 +160,11 @@ class Ticket:
                 limit_type = int.from_bytes(ticket_data.read(4))
                 limit_value = int.from_bytes(ticket_data.read(4))
                 self.title_limits_list.append(_TitleLimit(limit_type, limit_value))
+        # Check certs to see if this is a retail or dev ticket. Treats unknown certs as being retail for now.
+        if self.signature_issuer.find("Root-CA00000002-XS00000006") != -1:
+            self.is_dev = True
+        else:
+            self.is_dev = False
 
     def dump(self) -> bytes:
         """
@@ -315,7 +325,7 @@ class Ticket:
         bytes
             The decrypted title key.
         """
-        title_key = decrypt_title_key(self.title_key_enc, self.common_key_index, self.title_id)
+        title_key = decrypt_title_key(self.title_key_enc, self.common_key_index, self.title_id, self.is_dev)
         return title_key
 
     def set_title_id(self, title_id) -> None:

src/libWiiPy/title/title.py

@@ -77,7 +77,7 @@ class Title:
             self.wad.wad_type = "ib"
         # Dump the TMD and set it in the WAD.
         # This requires updating the content records and number of contents in the TMD first.
-        self.tmd.content_records = self.content.content_records
+        self.tmd.content_records = self.content.content_records  # This may not be needed because it's a ref already
         self.tmd.num_contents = len(self.content.content_records)
         self.wad.set_tmd_data(self.tmd.dump())
         # Dump the Ticket and set it in the WAD.
@@ -119,8 +119,9 @@ class Title:
         """
         if not self.tmd.content_records:
             ValueError("No TMD appears to have been loaded, so content records cannot be read from it.")
-        # Load the content records into the ContentRegion object.
+        # Load the content records into the ContentRegion object, and update the number of contents.
         self.content.content_records = self.tmd.content_records
+        self.content.num_contents = self.tmd.num_contents
 
     def set_title_id(self, title_id: str) -> None:
         """
@@ -137,7 +138,8 @@ class Title:
         self.tmd.set_title_id(title_id)
         title_key_decrypted = self.ticket.get_title_key()
         self.ticket.set_title_id(title_id)
-        title_key_encrypted = encrypt_title_key(title_key_decrypted, self.ticket.common_key_index, title_id)
+        title_key_encrypted = encrypt_title_key(title_key_decrypted, self.ticket.common_key_index, title_id,
+                                                self.ticket.is_dev)
         self.ticket.title_key_enc = title_key_encrypted
 
     def set_title_version(self, title_version: str | int) -> None:
@@ -296,13 +298,9 @@ class Title:
     def set_enc_content(self, enc_content: bytes, index: int, content_size: int, content_hash: bytes, cid: int = None,
                         content_type: int = None) -> None:
         """
-        Sets the content at the provided content index to the provided new encrypted content. The provided hash and
+        Sets the content at the provided index to the provided new encrypted content. The provided hash and content size
-        content size are set in the corresponding content record. A new Content ID or content type can also be
+        are set in the corresponding content record. A new Content ID or content type can also be specified, but if it
-        specified, but if it isn't then the current values are preserved.
+        isn't then the current values are preserved.
-
-        This uses the content index, which is the value tied to each content and used as the IV for encryption, rather
-        than the literal index in the array of content, because sometimes the contents end up out of order in a WAD
-        while still retaining the original indices.
 
         This also updates the content records in the TMD after the content is set.
 
@@ -328,9 +326,9 @@ class Title:
 
     def set_content(self, dec_content: bytes, index: int, cid: int = None, content_type: int = None) -> None:
         """
-        Sets the content at the provided content index to the provided new decrypted content. The hash and content size
+        Sets the content at the provided index to the provided new decrypted content. The hash and content size of this
-        of this content will be generated and then set in the corresponding content record. A new Content ID or content
+        content will be generated and then set in the corresponding content record. A new Content ID or content type can
-        type can also be specified, but if it isn't then the current values are preserved.
+        also be specified, but if it isn't then the current values are preserved.
 
         This also updates the content records in the TMD after the content is set.
 
@@ -356,16 +354,12 @@ class Title:
         sure that it matches the corresponding record. This content will then be encrypted using the title's Title Key
         before being loaded.
 
-        This uses the content index, which is the value tied to each content and used as the IV for encryption, rather
-        than the literal index in the array of content, because sometimes the contents end up out of order in a WAD
-        while still retaining the original indices.
-
         Parameters
         ----------
         dec_content : bytes
             The decrypted content to load.
         index : int
-            The content index to load the content at.
+            The index to load the content at.
         """
         # Load the decrypted content.
         self.content.load_content(dec_content, index, self.ticket.get_title_key())
@@ -382,6 +376,7 @@ class Title:
         after any changes to the TMD or Ticket, and before dumping the Title object into a WAD to ensure that the WAD
         is properly fakesigned.
         """
+        self.tmd.num_contents = self.content.num_contents  # This needs to be updated in case it was changed
         self.tmd.fakesign()
         self.ticket.fakesign()