Merge pull request #341 from harfbuzz/gen-tag-table.py

Switch more table generation to HarfBuzz

Author: behdad

harfrust/src/hb/ot_shaper_use_table.rs

@@ -1,4 +1,5 @@
-// WARNING: this file was generated by scripts/gen-vowel-constraints.py
+/* == Start of generated functions == */
+// WARNING: this file was generated by ../../harfbuzz/src/gen-vowel-constraints.py --rust
 
 #![allow(clippy::single_match)]
 
@@ -458,3 +459,4 @@ pub fn preprocess_text_vowel_constraints(buffer: &mut hb_buffer_t) {
     }
     buffer.sync();
 }
+/* == End of generated functions == */

harfrust/src/hb/ot_shaper_vowel_constraints.rs

@@ -139,6 +139,34 @@ fn lang_cmp(s1: &str, s2: &str) -> core::cmp::Ordering {
     s1[..ea].cmp(&s2[..eb])
 }
 
+pub(super) fn subtag_matches(language: &str, subtag: &str) -> bool {
+    for (i, _) in language.match_indices(subtag) {
+        if let Some(c) = language.as_bytes().get(i + subtag.len()) {
+            if !c.is_ascii_alphanumeric() {
+                return true;
+            }
+        } else {
+            return true;
+        }
+    }
+
+    false
+}
+
+pub(super) fn lang_matches(language: &str, spec: &str) -> bool {
+    if language.starts_with(spec) {
+        return language.len() == spec.len() || language.as_bytes().get(spec.len()) == Some(&b'-');
+    }
+
+    false
+}
+
+pub(super) fn strncmp(s1: &str, s2: &str, n: usize) -> bool {
+    let n1 = core::cmp::min(n, s1.len());
+    let n2 = core::cmp::min(n, s2.len());
+    s1[..n1] == s2[..n2]
+}
+
 fn tags_from_language(language: &Language, tags: &mut ThreeTags) {
     let language = language.as_str();
 

harfrust/src/hb/tag.rs

@@ -4,57 +4,30 @@
 repo. The Unicode/OT table generators shared with HarfBuzz live in `~/harfbuzz/src`
 and should be run from there.
 
+For regenerating the Rust-side shared Unicode/OT tables, use the wrapper
+makefile in this directory. It assumes HarfBuzz lives in `~/harfbuzz` by
+default and can be overridden with `HARFBUZZ_DIR=...`.
+
 Local generators in this repo:
 
 ```sh
 bash bump-version.sh 0.6.0
 
-python3 ./gen-vowel-constraints.py > ../harfrust/src/hb/ot_shaper_vowel_constraints.rs
-rustfmt ../harfrust/src/hb/ot_shaper_vowel_constraints.rs
+python3 ./gen-shaping-tests.py
 
-python3 ./gen-tag-table.py > ../harfrust/src/hb/tag_table.rs
-rustfmt ../harfrust/src/hb/tag_table.rs
+python3 ./gen-shaping-tests.py /path/to/harfbuzz
 ```
 
+`gen-shaping-tests.py` assumes HarfBuzz is in `~/harfbuzz` by default. If
+`~/harfbuzz/builddir/util/hb-shape` is missing but `~/harfbuzz/builddir`
+exists, it will run `ninja -C ~/harfbuzz/builddir` first.
+
 HarfBuzz-owned generators used by `harfrust`:
 
 ```sh
-PYTHONPATH=/Users/behdad/packtab \
-python3 ~/harfbuzz/src/gen-ucd-table.py --rust \
-  ~/harfbuzz/src/ucd.nounihan.grouped.xml \
-  ~/harfbuzz/src/hb-script-list.h \
-  > ../harfrust/src/hb/ucd_table.rs
-
-PYTHONPATH=/Users/behdad/packtab \
-python3 ~/harfbuzz/src/gen-use-table.py --rust \
-  ~/harfbuzz/src/IndicSyllabicCategory.txt \
-  ~/harfbuzz/src/IndicPositionalCategory.txt \
-  ~/harfbuzz/src/ArabicShaping.txt \
-  ~/harfbuzz/src/DerivedCoreProperties.txt \
-  ~/harfbuzz/src/UnicodeData.txt \
-  ~/harfbuzz/src/Blocks.txt \
-  ~/harfbuzz/src/Scripts.txt \
-  ./ms-use/IndicSyllabicCategory-Additional.txt \
-  ./ms-use/IndicPositionalCategory-Additional.txt \
-  > ../harfrust/src/hb/ot_shaper_use_table.rs
-
-PYTHONPATH=/Users/behdad/packtab \
-python3 ~/harfbuzz/src/gen-arabic-table.py --rust \
-  ~/harfbuzz/src/ArabicShaping.txt \
-  ~/harfbuzz/src/UnicodeData.txt \
-  ~/harfbuzz/src/Blocks.txt \
-  > ../harfrust/src/hb/ot_shaper_arabic_table.rs
-
-PYTHONPATH=/Users/behdad/packtab \
-python3 ~/harfbuzz/src/gen-indic-table.py --rust \
-  ~/harfbuzz/src/IndicSyllabicCategory.txt \
-  ~/harfbuzz/src/IndicPositionalCategory.txt \
-  ~/harfbuzz/src/Blocks.txt \
-  > ../harfrust/src/hb/ot_shaper_indic_table.rs
-
-PYTHONPATH=/Users/behdad/packtab \
-python3 ~/harfbuzz/src/gen-emoji-table.py --rust \
-  ~/harfbuzz/src/emoji-data.txt \
-  ~/harfbuzz/src/emoji-test.txt \
-  > ../harfrust/src/hb/unicode_emoji_table.rs
+make -f update-unicode-tables.mk
+
+make -f update-unicode-tables.mk hb-refresh
+
+make -f update-unicode-tables.mk HARFBUZZ_DIR=/path/to/harfbuzz
 ```

harfrust/src/hb/tag_table.rs

@@ -36,12 +36,21 @@
 ]
 
 
-def check_hb_build(hb_shape_exe):
-    if not hb_shape_exe.exists():
-        print("Build harfbuzz first using:")
-        print("    meson builddir")
-        print("    ninja -Cbuilddir")
-        exit(1)
+def check_hb_build(hb_dir, hb_shape_exe):
+    if hb_shape_exe.exists():
+        return
+
+    build_dir = hb_dir / "builddir"
+    if build_dir.exists():
+        subprocess.run(["ninja", "-C", str(build_dir)], check=True)
+        if hb_shape_exe.exists():
+            return
+
+    print("Build harfbuzz first using:")
+    print(f"    cd {hb_dir}")
+    print("    meson setup builddir")
+    print("    ninja -C builddir")
+    exit(1)
 
 
 def update_font_path(tests_name, fontfile):
@@ -233,18 +242,18 @@ def convert_test_files(root_dir, hb_shape_exe, tests_dir, tests_name, files, cus
 
 
 def main():
-    if len(sys.argv) != 2:
-        print("Usage: gen-shaping-tests.py /path/to/harfbuzz-src")
+    if len(sys.argv) > 2:
+        print("Usage: gen-shaping-tests.py [/path/to/harfbuzz-src]")
         exit(1)
 
-    hb_dir = Path(sys.argv[1])
+    hb_dir = Path(sys.argv[1]).expanduser() if len(sys.argv) == 2 else Path("~/harfbuzz").expanduser()
     assert hb_dir.exists()
 
     rb_root = pathlib.Path(__file__).parent.parent
 
     # Check that harfbuzz was built.
     hb_shape_exe = hb_dir.joinpath("builddir/util/hb-shape")
-    check_hb_build(hb_shape_exe)
+    check_hb_build(hb_dir, hb_shape_exe)
 
     def to_hb_absolute(name):
         return hb_dir / f"test/shape/data/{name}/tests"