add tests with UTF8/UTF16 non-ASCII text

PiperOrigin-RevId: 545424981

.github/workflows/codeql.yml

@@ -34,9 +34,12 @@ jobs:
       uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
+        # CodeQL is currently crashing on files with large lists:
+        #   https://github.com/github/codeql/issues/13656
         config: |
-          paths-ignore: 
+          paths-ignore:
             - research
+            - js/test_data.*
 
     - if: matrix.language == 'cpp'
       name: Build CPP

python/tests/_test_utils.py

@@ -45,6 +45,8 @@ TESTDATA_FILES = [
     'random_org_10k.bin',  # Small data
     'mapsdatazrh',  # Large data
     'ukkonooa',  # Poem
+    'cp1251-utf16le',  # Codepage 1251 table saved in UTF16-LE encoding
+    'cp852-utf8',  # Codepage 852 table saved in UTF8 encoding
 ]
 
 # Some files might be missing in a lightweight sources pack.