Add LoadCodepoints::BuiltInFrequenciesList()

It provides a list of available frequency data sets + the set of codepoints each one covers.

Author: garretrieger

MODULE.bazel

@@ -19,7 +19,7 @@ bazel_dep(name = "brotli", version = "1.2.0")
 
 # Frequency Data
 bazel_dep(name = "ift_encoder_data", version = "git")
-git_override(module_name = "ift_encoder_data", remote = "https://github.com/w3c/ift-encoder-data.git", commit = "317ea02ac68d45004aa842f831c65ed33c891701")
+git_override(module_name = "ift_encoder_data", remote = "https://github.com/w3c/ift-encoder-data.git", commit = "8f50ef507feac8f8bdf6c39a92f95b7c991dcfda")
 
 # Non Bazel Modules
 http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

util/BUILD

@@ -22,17 +22,6 @@ cc_proto_library(
     ],
 )
 
-proto_library(
-    name = "unicode_count_proto",
-    srcs = ["unicode_count.proto"],
-)
-
-cc_proto_library(
-    name = "unicode_count_cc_proto",
-    visibility = ["//visibility:public"],
-    deps = [":unicode_count_proto"],
-)
-
 proto_library(
     name = "common_proto",
     srcs = ["common.proto"],
@@ -178,12 +167,13 @@ cc_library(
         "load_codepoints.h",
     ],
     deps = [
-        ":unicode_count_cc_proto",
         "//common",
         "//ift/freq",
         "@abseil-cpp//absl/status:statusor",
         "@abseil-cpp//absl/strings",
         "@harfbuzz",
+        "@ift_encoder_data//:codepoint_count_cc_proto",
+        "@ift_encoder_data//:metadata_cc_proto",
         "@riegeli//riegeli/bytes:fd_reader",
         "@riegeli//riegeli/records:record_reader",
     ],
@@ -284,12 +274,12 @@ cc_binary(
         "generate_riegeli_test_data.cc",
     ],
     deps = [
-        ":unicode_count_cc_proto",
         "@abseil-cpp//absl/flags:flag",
         "@abseil-cpp//absl/flags:parse",
         "@abseil-cpp//absl/status",
         "@riegeli//riegeli/bytes:fd_writer",
         "@riegeli//riegeli/records:record_writer",
+        "@ift_encoder_data//:codepoint_count_cc_proto",
     ],
 )
 

util/generate_riegeli_test_data.cc

@@ -9,9 +9,10 @@
 #include "absl/strings/str_cat.h"
 #include "riegeli/bytes/fd_writer.h"
 #include "riegeli/records/record_writer.h"
-#include "util/unicode_count.pb.h"
+#include "codepoint_count.pb.h"
 
 using absl::StrCat;
+using ift_encoder_data::CodepointCount;
 
 ABSL_FLAG(std::string, output_path, "", "Path to write the output file.");
 

util/load_codepoints.cc

@@ -1,6 +1,5 @@
 #include "load_codepoints.h"
 
-#include <algorithm>
 #include <filesystem>
 #include <fstream>
 #include <iostream>
@@ -11,20 +10,26 @@
 #include "absl/strings/str_cat.h"
 #include "absl/strings/strip.h"
 #include "common/font_data.h"
+#include "common/int_set.h"
 #include "hb.h"
 #include "ift/freq/unicode_frequencies.h"
 #include "riegeli/bytes/fd_reader.h"
 #include "riegeli/records/record_reader.h"
-#include "util/unicode_count.pb.h"
+#include "codepoint_count.pb.h"
+#include "metadata.pb.h"
 
 using absl::Status;
 using absl::StatusOr;
 using absl::StrCat;
 using absl::string_view;
+using absl::flat_hash_map;
+using common::CodepointSet;
 using common::FontData;
 using common::hb_blob_unique_ptr;
 using common::make_hb_blob;
 using ift::freq::UnicodeFrequencies;
+using ift_encoder_data::CodepointCount;
+using ift_encoder_data::DatasetMetadata;
 
 namespace util {
 
@@ -210,4 +215,27 @@ StatusOr<UnicodeFrequencies> LoadBuiltInFrequencies(const char* name) {
   return LoadFrequenciesFromRiegeli(path.c_str());
 }
 
+StatusOr<flat_hash_map<std::string, common::CodepointSet>>
+BuiltInFrequenciesList() {
+  std::string path = "../ift_encoder_data+/data/metadata.binpb";
+  std::ifstream in(path, std::ios::binary);
+  if (!in.is_open()) {
+    return absl::NotFoundError(StrCat("Metadata file ", path, " was not found."));
+  }
+  DatasetMetadata metadata;
+  if (!metadata.ParseFromIstream(&in)) {
+    return absl::InternalError(StrCat("Failed to parse metadata file ", path));
+  }
+
+  flat_hash_map<std::string, common::CodepointSet> result;
+  for (const auto& file : metadata.files()) {
+    CodepointSet codepoints;
+    for (uint32_t cp : file.codepoints()) {
+      codepoints.insert(cp);
+    }
+    result[file.file_name()] = std::move(codepoints);
+  }
+  return result;
+}
+
 }  // namespace util

util/load_codepoints.h

@@ -5,6 +5,7 @@
 #include <vector>
 
 #include "absl/container/btree_set.h"
+#include "absl/container/flat_hash_map.h"
 #include "absl/status/statusor.h"
 #include "common/font_data.h"
 #include "common/font_helper.h"
@@ -43,13 +44,18 @@ absl::StatusOr<common::FontData> LoadFile(const char* path);
 absl::StatusOr<ift::freq::UnicodeFrequencies> LoadFrequenciesFromRiegeli(
     const char* path);
 
-// Loads frequency data from https://github.com/w3c/ift-encoder-data
+// loads frequency data from https://github.com/w3c/ift-encoder-data
 //
 // name is the file name to load.
 // Append "@*" to the name to load all sharded files for a name.
 absl::StatusOr<ift::freq::UnicodeFrequencies> LoadBuiltInFrequencies(
     const char* name);
 
+// Returns a list of all built-in frequency data sets and the codepoints
+// they cover.
+absl::StatusOr<absl::flat_hash_map<std::string, common::CodepointSet>>
+BuiltInFrequenciesList();
+
 // Given a filepath if it ends with @* this will expand the path into
 // the list of paths matching the pattern: <path>-?????-of-?????
 // Otherwise returns just the input path.

util/load_codepoints_test.cc

@@ -164,4 +164,13 @@ TEST_F(LoadCodepointsTest, LoadBuiltInFrequencies) {
   EXPECT_EQ(result->CoveredCodepoints().size(), 1363);
 }
 
+TEST_F(LoadCodepointsTest, BuiltInFrequenciesList) {
+  auto result = util::BuiltInFrequenciesList();
+  ASSERT_TRUE(result.ok()) << result.status();
+  EXPECT_FALSE(result->empty());
+  EXPECT_TRUE(result->contains("Script_latin.riegeli"));
+  EXPECT_FALSE((*result)["Script_latin.riegeli"].empty());
+  EXPECT_TRUE((*result)["Script_latin.riegeli"].contains('Q'));
+}
+
 }  // namespace util