Update Netflix

Add support for `{randomchar_N}` placeholders in Netflix Android `esn_map` values and generate those segments at runtime. Reuse a cached ESN only when it matches the derived template pattern, is Android-typed, and is not expired; otherwise regenerate and refresh the cache.

This keeps static ESN mappings working as before while enabling dynamic ESN templates (e.g., system_id `7110`) to avoid fixed identifiers and keep ESNs valid per template.

.gitignore

@@ -6,8 +6,6 @@ update_check.json
 *.exe
 *.dll
 *.crt
-*.wvd
-*.prd
 *.der
 *.pem
 *.bin
@@ -21,12 +19,11 @@ device_vmp_blob
 unshackle/cache/
 unshackle/cookies/
 unshackle/certs/
-unshackle/WVDs/
-unshackle/PRDs/
 temp/
 logs/
 Temp/
 binaries/
+Logs
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

unshackle/services/Netflix/MSL/__init__.py

@@ -10,7 +10,7 @@ import time
 import zlib
 from datetime import datetime
 from io import BytesIO
-from typing import Optional, Any
+from typing import Any, Optional
 
 import jsonpickle
 import requests
@@ -19,15 +19,18 @@ from Cryptodome.Hash import HMAC, SHA256
 from Cryptodome.PublicKey import RSA
 from Cryptodome.Random import get_random_bytes
 from Cryptodome.Util import Padding
+from pywidevine import PSSH, Cdm, Key
 
 from unshackle.core.cacher import Cacher
 
 from .MSLKeys import MSLKeys
-from .schemes import EntityAuthenticationSchemes  # noqa: F401
-from .schemes import KeyExchangeSchemes
+from .schemes import (
+    EntityAuthenticationSchemes,  # noqa: F401
+    KeyExchangeSchemes,
+)
 from .schemes.EntityAuthentication import EntityAuthentication
 from .schemes.KeyExchangeRequest import KeyExchangeRequest
-from pywidevine import Cdm, PSSH, Key
+
 
 class MSL:
     log = logging.getLogger("MSL")
@@ -41,7 +44,16 @@ class MSL:
         self.message_id = message_id
 
     @classmethod
-    def handshake(cls, scheme: KeyExchangeSchemes, session: requests.Session, endpoint: str, sender: str, cache: Cacher, cdm: Optional[Cdm] = None, config: Any = None):
+    def handshake(
+        cls,
+        scheme: KeyExchangeSchemes,
+        session: requests.Session,
+        endpoint: str,
+        sender: str,
+        cache: Cacher,
+        cdm: Optional[Cdm] = None,
+        config: Any = None,
+    ):
         cache = cache.get(sender)
         message_id = random.randint(0, pow(2, 52))
         msl_keys = MSL.load_cache_data(cache)
@@ -53,20 +65,18 @@ class MSL:
             if scheme != KeyExchangeSchemes.Widevine:
                 msl_keys.rsa = RSA.generate(2048)
 
-
             if scheme == KeyExchangeSchemes.Widevine:
                 if not cdm:
-                    raise Exception('Key exchange scheme Widevine but CDM instance is None.')
+                    raise Exception("Key exchange scheme Widevine but CDM instance is None.")
 
                 session_id = cdm.open()
                 msl_keys.cdm_session = session_id
                 cdm.set_service_certificate(session_id, config["certificate"])
                 challenge = cdm.get_license_challenge(
-                    session_id=session_id,
-                    pssh=PSSH("AAAANHBzc2gAAAAA7e+LqXnWSs6jyCfc1R0h7QAAABQIARIQAAAAAAPSZ0kAAAAAAAAAAA=="),
-                    license_type="OFFLINE",
-                    privacy_mode=True,
-
+                    session_id,
+                    PSSH("AAAANHBzc2gAAAAA7e+LqXnWSs6jyCfc1R0h7QAAABQIARIQAAAAAAPSZ0kAAAAAAAAAAA=="),
+                    "OFFLINE",
+                    True,
                 )
                 keyrequestdata = KeyExchangeRequest.Widevine(challenge)
                 entityauthdata = EntityAuthentication.Unauthenticated(sender)
@@ -76,47 +86,48 @@ class MSL:
                 keyrequestdata = KeyExchangeRequest.AsymmetricWrapped(
                     keypairid="superKeyPair",
                     mechanism="JWK_RSA",
-                    publickey=msl_keys.rsa.publickey().exportKey(format="DER")
+                    publickey=msl_keys.rsa.publickey().exportKey(format="DER"),
                 )
 
-            data = jsonpickle.encode({
+            data = jsonpickle.encode(
+                {
                     "entityauthdata": entityauthdata,
-                "headerdata": base64.b64encode(MSL.generate_msg_header(
-                    message_id=message_id,
-                    sender=sender,
-                    is_handshake=True,
-                    keyrequestdata=keyrequestdata
-                ).encode("utf-8")).decode("utf-8"),
-                "signature": ""
-            }, unpicklable=False)
-            data += json.dumps({
-                "payload": base64.b64encode(json.dumps({
-                    "messageid": message_id,
-                    "data": "",
-                    "sequencenumber": 1,
-                    "endofmsg": True
-                }).encode("utf-8")).decode("utf-8"),
-                "signature": ""
-            })
+                    "headerdata": base64.b64encode(
+                        MSL.generate_msg_header(
+                            message_id=message_id, sender=sender, is_handshake=True, keyrequestdata=keyrequestdata
+                        ).encode("utf-8")
+                    ).decode("utf-8"),
+                    "signature": "",
+                },
+                unpicklable=False,
+            )
+            data += json.dumps(
+                {
+                    "payload": base64.b64encode(
+                        json.dumps({"messageid": message_id, "data": "", "sequencenumber": 1, "endofmsg": True}).encode(
+                            "utf-8"
+                        )
+                    ).decode("utf-8"),
+                    "signature": "",
+                }
+            )
 
             try:
-                r = session.post(
-                    url=endpoint,
-                    data=data
-                )
+                r = session.post(url=endpoint, data=data)
             except requests.HTTPError as e:
                 raise Exception(f"- Key exchange failed, response data is unexpected: {e.response.text}")
 
             key_exchange = r.json()  # expecting no payloads, so this is fine
             if "errordata" in key_exchange:
-                raise Exception("- Key exchange failed: " + json.loads(base64.b64decode(
-                    key_exchange["errordata"]
-                ).decode())["errormsg"])
+                raise Exception(
+                    "- Key exchange failed: "
+                    + json.loads(base64.b64decode(key_exchange["errordata"]).decode())["errormsg"]
+                )
 
             # parse the crypto keys
-            key_response_data = json.JSONDecoder().decode(base64.b64decode(
-                key_exchange["headerdata"]
-            ).decode("utf-8"))["keyresponsedata"]
+            key_response_data = json.JSONDecoder().decode(base64.b64decode(key_exchange["headerdata"]).decode("utf-8"))[
+                "keyresponsedata"
+            ]
 
             if key_response_data["scheme"] != str(scheme):
                 raise Exception("- Key exchange scheme mismatch occurred")
@@ -129,46 +140,40 @@ class MSL:
                     raise Exception("- No CDM available")
                 cdm.parse_license(msl_keys.cdm_session, key_data["cdmkeyresponse"])
                 keys = cdm.get_keys(msl_keys.cdm_session)
-                cls.log.info(f"Keys: {keys}")
+                cls.log.debug(f"Keys: {keys}")
                 encryption_key = MSL.get_widevine_key(
                     kid=base64.b64decode(key_data["encryptionkeyid"]),
                     keys=keys,
-                    permissions=["allow_encrypt", "allow_decrypt"]
+                    permissions=["allow_encrypt", "allow_decrypt"],
                 )
                 msl_keys.encryption = encryption_key
-                cls.log.info(f"Encryption key: {encryption_key}")
+                cls.log.debug(f"Encryption key: {encryption_key}")
                 sign = MSL.get_widevine_key(
                     kid=base64.b64decode(key_data["hmackeyid"]),
                     keys=keys,
-                    permissions=["allow_sign", "allow_signature_verify"]
+                    permissions=["allow_sign", "allow_signature_verify"],
                 )
-                cls.log.info(f"Sign key: {sign}")
+                cls.log.debug(f"Sign key: {sign}")
                 msl_keys.sign = sign
 
             elif scheme == KeyExchangeSchemes.AsymmetricWrapped:
                 cipher_rsa = PKCS1_OAEP.new(msl_keys.rsa)
                 msl_keys.encryption = MSL.base64key_decode(
-                    json.JSONDecoder().decode(cipher_rsa.decrypt(
-                        base64.b64decode(key_data["encryptionkey"])
-                    ).decode("utf-8"))["k"]
+                    json.JSONDecoder().decode(
+                        cipher_rsa.decrypt(base64.b64decode(key_data["encryptionkey"])).decode("utf-8")
+                    )["k"]
                 )
                 msl_keys.sign = MSL.base64key_decode(
-                    json.JSONDecoder().decode(cipher_rsa.decrypt(
-                        base64.b64decode(key_data["hmackey"])
-                    ).decode("utf-8"))["k"]
+                    json.JSONDecoder().decode(
+                        cipher_rsa.decrypt(base64.b64decode(key_data["hmackey"])).decode("utf-8")
+                    )["k"]
                 )
 
             msl_keys.mastertoken = key_response_data["mastertoken"]
 
             MSL.cache_keys(msl_keys, cache)
             cls.log.info("MSL handshake successful")
-        return cls(
-            session=session,
-            endpoint=endpoint,
-            sender=sender,
-            keys=msl_keys,
-            message_id=message_id
-        )
+        return cls(session=session, endpoint=endpoint, sender=sender, keys=msl_keys, message_id=message_id)
 
     @staticmethod
     def load_cache_data(cacher: Cacher):
@@ -184,9 +189,24 @@ class MSL:
             # to an RsaKey :)
             msl_keys.rsa = RSA.importKey(msl_keys.rsa)
         # If it's expired or close to, return None as it's unusable
-        if msl_keys.mastertoken and ((datetime.utcfromtimestamp(int(json.JSONDecoder().decode(
+        if (
+            msl_keys.mastertoken
+            and (
+                (
+                    datetime.utcfromtimestamp(
+                        int(
+                            json.JSONDecoder().decode(
                                 base64.b64decode(msl_keys.mastertoken["tokendata"]).decode("utf-8")
-        )["expiration"])) - datetime.now()).total_seconds() / 60 / 60) < 10:
+                            )["expiration"]
+                        )
+                    )
+                    - datetime.now()
+                ).total_seconds()
+                / 60
+                / 60
+            )
+            < 10
+        ):
             return None
         return msl_keys
 
@@ -204,8 +224,9 @@ class MSL:
             msl_keys.rsa = RSA.importKey(msl_keys.rsa)
 
     @staticmethod
-    def generate_msg_header(message_id, sender, is_handshake, userauthdata=None, keyrequestdata=None,
-                            compression="GZIP"):
+    def generate_msg_header(
+        message_id, sender, is_handshake, userauthdata=None, keyrequestdata=None, compression="GZIP"
+    ):
         """
         The MSL header carries all MSL data used for entity and user authentication, message encryption
         and verification, and service tokens. Portions of the MSL header are encrypted.
@@ -228,7 +249,7 @@ class MSL:
             "capabilities": {
                 "compressionalgos": [compression] if compression else [],
                 "languages": ["en-US"],  # bcp-47
-                "encoderformats": ["JSON"]
+                "encoderformats": ["JSON"],
             },
             "timestamp": int(time.time()),
             # undocumented or unused:
@@ -244,7 +265,7 @@ class MSL:
 
     @classmethod
     def get_widevine_key(cls, kid, keys: list[Key], permissions):
-        cls.log.info(f"KID: {Key.kid_to_uuid(kid)}")
+        cls.log.debug(f"KID: {Key.kid_to_uuid(kid)}")
         for key in keys:
             # cls.log.info(f"KEY: {key.kid_to_uuid}")
             if key.kid != Key.kid_to_uuid(kid):
@@ -258,10 +279,10 @@ class MSL:
             return key.key
         return None
 
-    def send_message(self, endpoint, params, application_data, userauthdata=None):
+    def send_message(self, endpoint, params, application_data, userauthdata=None, headers=None, unwrap_result=True):
         message = self.create_message(application_data, userauthdata)
-        res = self.session.post(url=endpoint, data=message, params=params)
-        header, payload_data = self.parse_message(res.text)
+        res = self.session.post(url=endpoint, data=message, params=params, headers=headers)
+        header, payload_data = self.parse_message(res.text, unwrap_result=unwrap_result)
         if "errordata" in header:
             raise Exception(
                 "- MSL response message contains an error: {}".format(
@@ -272,37 +293,46 @@ class MSL:
 
     def create_message(self, application_data, userauthdata=None):
         self.message_id += 1  # new message must ue a new message id
-        headerdata = self.encrypt(self.generate_msg_header(
-            message_id=self.message_id,
-            sender=self.sender,
-            is_handshake=False,
-            userauthdata=userauthdata
-        ))
+        headerdata = self.encrypt(
+            self.generate_msg_header(
+                message_id=self.message_id, sender=self.sender, is_handshake=False, userauthdata=userauthdata
+            )
+        )
 
-        header = json.dumps({
+        header = json.dumps(
+            {
                 "headerdata": base64.b64encode(headerdata.encode("utf-8")).decode("utf-8"),
                 "signature": self.sign(headerdata).decode("utf-8"),
-            "mastertoken": self.keys.mastertoken
-        })
+                "mastertoken": self.keys.mastertoken,
+            }
+        )
 
-        payload_chunks = [self.encrypt(json.dumps({
+        payload_chunks = [
+            self.encrypt(
+                json.dumps(
+                    {
                         "messageid": self.message_id,
                         "data": self.gzip_compress(json.dumps(application_data).encode("utf-8")).decode("utf-8"),
                         "compressionalgo": "GZIP",
                         "sequencenumber": 1,  # todo ; use sequence_number from master token instead?
-            "endofmsg": True
-        }))]
+                        "endofmsg": True,
+                    }
+                )
+            )
+        ]
 
         message = header
         for payload_chunk in payload_chunks:
-            message += json.dumps({
+            message += json.dumps(
+                {
                     "payload": base64.b64encode(payload_chunk.encode("utf-8")).decode("utf-8"),
-                "signature": self.sign(payload_chunk).decode("utf-8")
-            })
+                    "signature": self.sign(payload_chunk).decode("utf-8"),
+                }
+            )
 
         return message
 
-    def decrypt_payload_chunks(self, payload_chunks):
+    def decrypt_payload_chunks(self, payload_chunks, unwrap_result=True):
         """
         Decrypt and extract data from payload chunks
 
@@ -310,16 +340,13 @@ class MSL:
         :return: json object
         """
         raw_data = ""
-
         for payload_chunk in payload_chunks:
             # todo ; verify signature of payload_chunk["signature"] against payload_chunk["payload"]
             # expecting base64-encoded json string
             payload_chunk = json.loads(base64.b64decode(payload_chunk["payload"]).decode("utf-8"))
             # decrypt the payload
             payload_decrypted = AES.new(
-                key=self.keys.encryption,
-                mode=AES.MODE_CBC,
-                iv=base64.b64decode(payload_chunk["iv"])
+                key=self.keys.encryption, mode=AES.MODE_CBC, iv=base64.b64decode(payload_chunk["iv"])
             ).decrypt(base64.b64decode(payload_chunk["ciphertext"]))
             payload_decrypted = Padding.unpad(payload_decrypted, 16)
             payload_decrypted = json.loads(payload_decrypted.decode("utf-8"))
@@ -344,10 +371,12 @@ class MSL:
                 self.log.critical(f"- {error}")
             raise Exception(f"- MSL response message contains an error: {error}")
             # sys.exit(1)
-
+        self.log.debug(f"Payload Chunks: {data}")
+        if unwrap_result:
             return data["result"]
+        return data
 
-    def parse_message(self, message):
+    def parse_message(self, message, unwrap_result=True):
         """
         Parse an MSL message into a header and list of payload chunks
 
@@ -359,7 +388,7 @@ class MSL:
         header = parsed_message[0]
         encrypted_payload_chunks = parsed_message[1:] if len(parsed_message) > 1 else []
         if encrypted_payload_chunks:
-            payload_chunks = self.decrypt_payload_chunks(encrypted_payload_chunks)
+            payload_chunks = self.decrypt_payload_chunks(encrypted_payload_chunks, unwrap_result=unwrap_result)
         else:
             payload_chunks = {}
 
@@ -390,22 +419,19 @@ class MSL:
         :return: Serialized JSON String of the encryption Envelope
         """
         iv = get_random_bytes(16)
-        return json.dumps({
+        return json.dumps(
+            {
                 "ciphertext": base64.b64encode(
-                AES.new(
-                    self.keys.encryption,
-                    AES.MODE_CBC,
-                    iv
-                ).encrypt(
-                    Padding.pad(plaintext.encode("utf-8"), 16)
-                )
+                    AES.new(self.keys.encryption, AES.MODE_CBC, iv).encrypt(Padding.pad(plaintext.encode("utf-8"), 16))
                 ).decode("utf-8"),
-            "keyid": "{}_{}".format(self.sender, json.loads(
-                base64.b64decode(self.keys.mastertoken["tokendata"]).decode("utf-8")
-            )["sequencenumber"]),
+                "keyid": "{}_{}".format(
+                    self.sender,
+                    json.loads(base64.b64decode(self.keys.mastertoken["tokendata"]).decode("utf-8"))["sequencenumber"],
+                ),
                 "sha256": "AA==",
-            "iv": base64.b64encode(iv).decode("utf-8")
-        })
+                "iv": base64.b64encode(iv).decode("utf-8"),
+            }
+        )
 
     def sign(self, text):
         """

unshackle/services/Netflix/MSL/schemes/UserAuthentication.py

@@ -31,6 +31,19 @@ class UserAuthentication(MSLObject):
             }
         )
 
+    @classmethod
+    def UserIDToken(cls, token_data, signature, master_token):
+        return cls(
+            scheme=UserAuthenticationSchemes.UserIDToken,
+            authdata={
+                "useridtoken": {
+                    "tokendata": token_data,
+                    "signature": signature
+                },
+                "mastertoken": master_token
+            }
+        )
+
     @classmethod
     def NetflixIDCookies(cls, netflixid, securenetflixid):
         """

unshackle/services/Netflix/MSL/schemes/__init__.py

@@ -15,6 +15,7 @@ class EntityAuthenticationSchemes(Scheme):
 class UserAuthenticationSchemes(Scheme):
     """https://github.com/Netflix/msl/wiki/User-Authentication-%28Configuration%29"""
     EmailPassword = "EMAIL_PASSWORD"
+    UserIDToken = "USER_ID_TOKEN"
     NetflixIDCookies = "NETFLIXID"
 
 

unshackle/unshackle.yaml

@@ -36,61 +36,32 @@ title_cache_max_retention: 86400 # Maximum cache retention for fallback when API
 muxing:
   set_title: true
 
+# Configuration for serve
+serve:
+  api_secret: "kenzuya"
+  users:
+    secret_key_for_user:
+      devices:
+        - generic_nexus_4464_l3
+      username: user
+
 # Login credentials for each Service
 credentials:
-  # Direct credentials (no profile support)
-  EXAMPLE: email@example.com:password
-
-  # Per-profile credentials with default fallback
-  SERVICE_NAME:
-    default: default@email.com:password # Used when no -p/--profile is specified
-    profile1: user1@email.com:password1
-    profile2: user2@email.com:password2
-
-  # Per-profile credentials without default (requires -p/--profile)
-  SERVICE_NAME2:
-    john: john@example.com:johnspassword
-    jane: jane@example.com:janespassword
-
-  # You can also use list format for passwords with special characters
-  SERVICE_NAME3:
-    default: ["user@email.com", ":PasswordWith:Colons"]
-
   Netflix:
     default: ["ariel-prinsess828@ezweb.ne.jp", "AiNe892186"]
+    secondary: ["csyc5478@naver.com", "wl107508!"]
+    third: ["erin.e.pfleger@gmail.com", "Pfleger93"]
     # default: ["pbgarena0838@gmail.com", "Andhika1978"]
 # Override default directories used across unshackle
 directories:
   cache: Cache
-  # cookies: Cookies
   dcsl: DCSL # Device Certificate Status List
-  downloads: Downloads
+  downloads: /mnt/ketuakenzuya/Downloads/
   logs: Logs
-  temp: Temp
-  # wvds: WVDs
-  # Additional directories that can be configured:
-  # commands: Commands
-  # services:
-  #   - /path/to/services
-  #   - /other/path/to/services
-  # vaults: Vaults
-  # fonts: Fonts
-
-# Pre-define which Widevine or PlayReady device to use for each Service
+  temp: /tmp/unshackle
 cdm:
-  # Global default CDM device (fallback for all services/profiles)
   default: chromecdm
 
-  # Direct service-specific CDM
-  DIFFERENT_EXAMPLE: PRD_1
-
-  # Per-profile CDM configuration
-  EXAMPLE:
-    john_sd: chromecdm_903_l3 # Profile 'john_sd' uses Chrome CDM L3
-    jane_uhd: nexus_5_l1 # Profile 'jane_uhd' uses Nexus 5 L1
-    default: generic_android_l3 # Default CDM for this service
-
-# Use pywidevine Serve-compliant Remote CDMs
 remote_cdm:
   - name: "chromecdm"
     device_name: widevine
@@ -99,7 +70,7 @@ remote_cdm:
     security_level: 3
     type: "decrypt_labs"
     host: https://keyxtractor.decryptlabs.com
-    secret: 7547150416_41da0a32d6237d83_KeyXtractor_api_ext
+    secret: 919240143_41d9c3fac9a5f82e_KeyXtractor_ultimate
   - name: "android"
     device_name: andorid
     device_type: ANDROID
@@ -127,22 +98,7 @@ key_vaults:
     api_mode: "decrypt_labs"
     host: "https://keyvault.decryptlabs.com"
     password: "7547150416_41da0a32d6237d83_KeyXtractor_api_ext"
-  # Additional vault types:
-  # - type: API
-  #   name: "Remote Vault"
-  #   uri: "https://key-vault.example.com"
-  #   token: "secret_token"
-  #   no_push: true  # This vault will only provide keys, not receive them
-  # - type: MySQL
-  #   name: "MySQL Vault"
-  #   host: "127.0.0.1"
-  #   port: 3306
-  #   database: vault
-  #   username: user
-  #   password: pass
-  #   no_push: false  # Default behavior - vault both provides and receives keys
 
-# Choose what software to use to download data
 downloader: aria2c
 # Options: requests | aria2c | curl_impersonate | n_m3u8dl_re
 # Can also be a mapping:
@@ -199,26 +155,10 @@ filenames:
 # API key for The Movie Database (TMDB)
 tmdb_api_key: "8f5c14ef648a0abdd262cf809e11fcd4"
 
-# conversion_method:
-# - auto (default): Smart routing - subby for WebVTT/SAMI, standard for others
-# - subby: Always use subby with advanced processing
-# - pycaption: Use only pycaption library (no SubtitleEdit, no subby)
-# - subtitleedit: Prefer SubtitleEdit when available, fall back to pycaption
 subtitle:
   conversion_method: auto
   sdh_method: auto
 
-# Configuration for pywidevine's serve functionality
-serve:
-  users:
-    secret_key_for_user:
-      devices:
-        - generic_nexus_4464_l3
-      username: user
-  # devices:
-  #   - '/path/to/device.wvd'
-
-# Configuration data for each Service
 services:
   # Service-specific configuration goes here
   # Profile-specific configurations can be nested under service names
@@ -251,6 +191,21 @@ services:
 
 # External proxy provider services
 proxy_providers:
+  gluetun:
+    base_port: 8888
+    auto_cleanup: true
+    container_prefix: "unshackle-gluetun"
+    verify_ip: true
+    providers:
+      protonvpn:
+        vpn_type: "openvpn"
+        credentials:
+          username: "L83JaCnXKIviymQm"
+          password: "UewUDYdthTLLhOBJDympFFxJn4uG12BV"
+        server_countries:
+          us: United States
+          id: Indonesia
+          kr: Korea
   basic:
     SG:
       - "http://127.0.0.1:6004"