.circleci/config.yml | 6 src/hb-blob.cc | 152 ++++------ test/api/fonts/clusterfuzz-testcase-minimized-hb-subset-fuzzer-5750092395970560 |binary test/subset/run-tests.py | 101 +++--- util/options.cc | 27 - 5 files changed, 134 insertions(+), 152 deletions(-)
New commits: commit 7db2e9ea38329b9393c9e8cc905b180735c9b0f4 Author: Ebrahim Byagowi <[email protected]> Date: Tue Jun 26 10:46:10 2018 +0430 Minor on hb_blob_create_from_file Add one more "unlikely" annotation and use explicit nullptr check for more consistency. diff --git a/src/hb-blob.cc b/src/hb-blob.cc index 37a6d8cb..155c2e6b 100644 --- a/src/hb-blob.cc +++ b/src/hb-blob.cc @@ -597,7 +597,7 @@ fail_without_close: int len = 0; int allocated = BUFSIZ * 16; char *blob = (char *) malloc (allocated); - if (blob == nullptr) return hb_blob_get_empty (); + if (unlikely (blob == nullptr)) return hb_blob_get_empty (); FILE *fp = fopen (file_name, "rb"); if (unlikely (fp == nullptr)) goto fread_fail_without_close; @@ -611,7 +611,7 @@ fail_without_close: // can cover files like that but lets limit our fallback reader if (unlikely (allocated > 200000000)) goto fread_fail; char *new_blob = (char *) realloc (blob, allocated); - if (unlikely (!new_blob)) goto fread_fail; + if (unlikely (new_blob == nullptr)) goto fread_fail; blob = new_blob; } commit 4f8753464ae44dfb60bee81ede10448175db7b90 Author: Garret Rieger <[email protected]> Date: Fri Jun 22 15:29:34 2018 -0700 [subset] Add fuzzer test case that caused a timeout to the corpus. diff --git a/test/api/fonts/clusterfuzz-testcase-minimized-hb-subset-fuzzer-5750092395970560 b/test/api/fonts/clusterfuzz-testcase-minimized-hb-subset-fuzzer-5750092395970560 new file mode 100644 index 00000000..d622c256 Binary files /dev/null and b/test/api/fonts/clusterfuzz-testcase-minimized-hb-subset-fuzzer-5750092395970560 differ commit 35ce8f31d37cf7c2a1f8265d36ba4c2c9a3efb2c Author: Ebrahim Byagowi <[email protected]> Date: Mon Jun 25 22:23:43 2018 +0430 Unify our pipe reader with the fallback reader (#1068) And assign one bot to use the path always using NOMMAPFILEREADER token. It's limited to 200mb so no more fun with using /dev/zero on hb-view! diff --git a/.circleci/config.yml b/.circleci/config.yml index 55820005..de130b50 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,14 +40,14 @@ jobs: - run: rm -rf harfbuzz-* - run: make distdir && cd harfbuzz-* && cmake -DHB_CHECK=ON -Bbuild -H. -GNinja && ninja -Cbuild && CTEST_OUTPUT_ON_FAILURE=1 ninja -Cbuild test && ninja -Cbuild install - alpine-O3: + alpine-O3-NOMMAP: docker: - image: alpine steps: - checkout - run: apk update && apk add ragel make pkgconfig libtool autoconf automake gettext gcc g++ glib-dev freetype-dev cairo-dev # C??FLAGS are not needed for a regular build - - run: CFLAGS="-O3" CXXFLAGS="-O3" ./autogen.sh + - run: CFLAGS="-O3" CXXFLAGS="-O3 -DNOMMAPFILEREADER" ./autogen.sh - run: make - run: make check || .ci/fail.sh @@ -196,7 +196,7 @@ workflows: - distcheck # autotools based builds - - alpine-O3 + - alpine-O3-NOMMAP - archlinux-debug-O0-py3 - clang-O3-O0 - fedora-outoftreebuild diff --git a/src/hb-blob.cc b/src/hb-blob.cc index 77dbc715..37a6d8cb 100644 --- a/src/hb-blob.cc +++ b/src/hb-blob.cc @@ -499,36 +499,6 @@ hb_blob_t::try_make_writable (void) # define MAP_NORESERVE 0 #endif -static hb_blob_t * -_fread_to_end (FILE *file) -{ - int allocated = BUFSIZ * 16; - char *blob = (char *) malloc (allocated); - if (blob == nullptr) return hb_blob_get_empty (); - - int len = 0; - while (!feof (file)) - { - if (allocated - len < BUFSIZ) - { - allocated *= 2; - char *new_blob = (char *) realloc (blob, allocated); - if (unlikely (!new_blob)) goto fail; - blob = new_blob; - } - - len += fread (blob + len, 1, allocated - len, file); - if (ferror (file)) goto fail; - } - - return hb_blob_create (blob, len, HB_MEMORY_MODE_WRITABLE, blob, - (hb_destroy_func_t) free); - -fail: - free (blob); - return hb_blob_get_empty (); -} - struct hb_mapped_file_t { char *contents; @@ -547,7 +517,7 @@ _hb_mapped_file_destroy (hb_mapped_file_t *file) UnmapViewOfFile (file->contents); CloseHandle (file->mapping); #else - free (file->contents); + assert (0); // If we don't have mmap we shouldn't reach here #endif free (file); @@ -566,87 +536,95 @@ hb_blob_create_from_file (const char *file_name) { // Adopted from glib's gmappedfile.c with Matthias Clasen and // Allison Lortie permission but changed a lot to suit our need. - bool writable = false; - hb_memory_mode_t mm = HB_MEMORY_MODE_READONLY_MAY_MAKE_WRITABLE; +#if defined(HAVE_MMAP) && !defined(NOMMAPFILEREADER) hb_mapped_file_t *file = (hb_mapped_file_t *) calloc (1, sizeof (hb_mapped_file_t)); if (unlikely (!file)) return hb_blob_get_empty (); -#ifdef HAVE_MMAP - int fd = open (file_name, (writable ? O_RDWR : O_RDONLY) | _O_BINARY, 0); -# define CLOSE close + int fd = open (file_name, O_RDONLY | _O_BINARY, 0); if (unlikely (fd == -1)) goto fail_without_close; struct stat st; if (unlikely (fstat (fd, &st) == -1)) goto fail; - // See https://github.com/GNOME/glib/blob/f9faac7/glib/gmappedfile.c#L139-L142 - if (unlikely (st.st_size == 0 && S_ISREG (st.st_mode))) goto fail; - file->length = (unsigned long) st.st_size; - file->contents = (char *) mmap (nullptr, file->length, - writable ? PROT_READ|PROT_WRITE : PROT_READ, + file->contents = (char *) mmap (nullptr, file->length, PROT_READ, MAP_PRIVATE | MAP_NORESERVE, fd, 0); - if (unlikely (file->contents == MAP_FAILED)) - { - free (file); - FILE *f = fdopen (fd, "rb"); - if (unlikely (f == nullptr)) - { - CLOSE (fd); - return hb_blob_get_empty (); - } - hb_blob_t *blob = _fread_to_end (f); - fclose (f); - return blob; - } + if (unlikely (file->contents == MAP_FAILED)) goto fail; -#elif defined(_WIN32) || defined(__CYGWIN__) - HANDLE fd = CreateFile (file_name, - writable ? GENERIC_READ|GENERIC_WRITE : GENERIC_READ, - FILE_SHARE_READ, nullptr, OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED, nullptr); -# define CLOSE CloseHandle + close (fd); + + return hb_blob_create (file->contents, file->length, + HB_MEMORY_MODE_READONLY_MAY_MAKE_WRITABLE, (void *) file, + (hb_destroy_func_t) _hb_mapped_file_destroy); + +fail: + close (fd); +fail_without_close: + free (file); + +#elif (defined(_WIN32) || defined(__CYGWIN__)) && !defined(NOMMAPFILEREADER) + hb_mapped_file_t *file = (hb_mapped_file_t *) calloc (1, sizeof (hb_mapped_file_t)); + if (unlikely (!file)) return hb_blob_get_empty (); + + HANDLE fd = CreateFile (file_name, GENERIC_READ, FILE_SHARE_READ, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL|FILE_FLAG_OVERLAPPED, + nullptr); if (unlikely (fd == INVALID_HANDLE_VALUE)) goto fail_without_close; file->length = (unsigned long) GetFileSize (fd, nullptr); - file->mapping = CreateFileMapping (fd, nullptr, - writable ? PAGE_WRITECOPY : PAGE_READONLY, - 0, 0, nullptr); + file->mapping = CreateFileMapping (fd, nullptr, PAGE_READONLY, 0, 0, nullptr); if (unlikely (file->mapping == nullptr)) goto fail; - file->contents = (char *) MapViewOfFile (file->mapping, - writable ? FILE_MAP_COPY : FILE_MAP_READ, - 0, 0, 0); + file->contents = (char *) MapViewOfFile (file->mapping, FILE_MAP_READ, 0, 0, 0); if (unlikely (file->contents == nullptr)) goto fail; -#else - mm = HB_MEMORY_MODE_WRITABLE; + CloseHandle (fd); + return hb_blob_create (file->contents, file->length, + HB_MEMORY_MODE_READONLY_MAY_MAKE_WRITABLE, (void *) file, + (hb_destroy_func_t) _hb_mapped_file_destroy); - FILE *fd = fopen (file_name, "rb"); -# define CLOSE fclose - if (unlikely (!fd)) goto fail_without_close; +fail: + CloseHandle (fd); +fail_without_close: + free (file); + +#endif + + // The following tries to read a file without knowing its size beforehand + // It's used for systems without mmap concept or to read from pipes + int len = 0; + int allocated = BUFSIZ * 16; + char *blob = (char *) malloc (allocated); + if (blob == nullptr) return hb_blob_get_empty (); - fseek (fd, 0, SEEK_END); - file->length = ftell (fd); - rewind (fd); - file->contents = (char *) malloc (file->length); - if (unlikely (!file->contents)) goto fail; + FILE *fp = fopen (file_name, "rb"); + if (unlikely (fp == nullptr)) goto fread_fail_without_close; - if (unlikely (fread (file->contents, 1, file->length, fd) != file->length)) - goto fail; + while (!feof (fp)) + { + if (allocated - len < BUFSIZ) + { + allocated *= 2; + // Don't allocate more than 200MB, our mmap reader still + // can cover files like that but lets limit our fallback reader + if (unlikely (allocated > 200000000)) goto fread_fail; + char *new_blob = (char *) realloc (blob, allocated); + if (unlikely (!new_blob)) goto fread_fail; + blob = new_blob; + } -#endif + len += fread (blob + len, 1, allocated - len, fp); + if (unlikely (ferror (fp))) goto fread_fail; + } - CLOSE (fd); - return hb_blob_create (file->contents, file->length, mm, (void *) file, - (hb_destroy_func_t) _hb_mapped_file_destroy); + return hb_blob_create (blob, len, HB_MEMORY_MODE_WRITABLE, blob, + (hb_destroy_func_t) free); -fail: - CLOSE (fd); -#undef CLOSE -fail_without_close: - free (file); +fread_fail: + fclose (fp); +fread_fail_without_close: + free (blob); return hb_blob_get_empty (); } commit f57804a8a596e88843ddc8b88afac7526349b89b Author: Ebrahim Byagowi <[email protected]> Date: Mon Jun 25 18:45:49 2018 +0430 Resolve ttx absolute path before use (#1075) diff --git a/test/subset/run-tests.py b/test/subset/run-tests.py index 1cd1a19f..bc0d082e 100755 --- a/test/subset/run-tests.py +++ b/test/subset/run-tests.py @@ -15,75 +15,94 @@ import tempfile from subset_test_suite import SubsetTestSuite +# https://stackoverflow.com/a/377028 +def which(program): + def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + fpath, _ = os.path.split(program) + if fpath: + if is_exe(program): + return program + else: + for path in os.environ["PATH"].split(os.pathsep): + exe_file = os.path.join(path, program) + if is_exe(exe_file): + return exe_file + + return None + +ttx = which ("ttx") +ots_sanitize = which ("ots-sanitize") + +if not ttx: + print("TTX is not present, skipping test.") + sys.exit (77) def cmd(command): p = subprocess.Popen ( command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdoutdata, stderrdata) = p.communicate() + (stdoutdata, stderrdata) = p.communicate () print (stderrdata, end="") # file=sys.stderr return stdoutdata, p.returncode -def read_binary(file_path): - with open(file_path, 'rb') as f: - return f.read() +def read_binary (file_path): + with open (file_path, 'rb') as f: + return f.read () def fail_test(test, cli_args, message): print ('ERROR: %s' % message) print ('Test State:') - print (' test.font_path %s' % os.path.abspath(test.font_path)) - print (' test.profile_path %s' % os.path.abspath(test.profile_path)) - print (' test.unicodes %s' % test.unicodes()) - expected_file = os.path.join(test_suite.get_output_directory(), - test.get_font_name()) - print (' expected_file %s' % os.path.abspath(expected_file)) + print (' test.font_path %s' % os.path.abspath (test.font_path)) + print (' test.profile_path %s' % os.path.abspath (test.profile_path)) + print (' test.unicodes %s' % test.unicodes ()) + expected_file = os.path.join(test_suite.get_output_directory (), + test.get_font_name ()) + print (' expected_file %s' % os.path.abspath (expected_file)) return 1 def run_test(test, should_check_ots): - out_file = os.path.join(tempfile.mkdtemp(), test.get_font_name() + '-subset.ttf') + out_file = os.path.join(tempfile.mkdtemp (), test.get_font_name () + '-subset.ttf') cli_args = [hb_subset, "--font-file=" + test.font_path, "--output-file=" + out_file, - "--unicodes=%s" % test.unicodes()] - cli_args.extend (test.get_profile_flags()) - print (' '.join(cli_args)) - _, return_code = cmd(cli_args) + "--unicodes=%s" % test.unicodes ()] + cli_args.extend (test.get_profile_flags ()) + print (' '.join (cli_args)) + _, return_code = cmd (cli_args) if return_code: - return fail_test(test, cli_args, "%s returned %d" % (' '.join(cli_args), return_code)) + return fail_test (test, cli_args, "%s returned %d" % (' '.join (cli_args), return_code)) - expected_ttx, return_code = run_ttx(os.path.join(test_suite.get_output_directory(), - test.get_font_name())) + expected_ttx, return_code = run_ttx (os.path.join (test_suite.get_output_directory (), + test.get_font_name ())) if return_code: - return fail_test(test, cli_args, "ttx (expected) returned %d" % (return_code)) + return fail_test (test, cli_args, "ttx (expected) returned %d" % (return_code)) actual_ttx, return_code = run_ttx(out_file) if return_code: - return fail_test(test, cli_args, "ttx (actual) returned %d" % (return_code)) + return fail_test (test, cli_args, "ttx (actual) returned %d" % (return_code)) print ("stripping checksums.") expected_ttx = strip_check_sum (expected_ttx) actual_ttx = strip_check_sum (actual_ttx) if not actual_ttx == expected_ttx: - for line in unified_diff(expected_ttx.splitlines(1), actual_ttx.splitlines(1)): - sys.stdout.write(line) - sys.stdout.flush() + for line in unified_diff (expected_ttx.splitlines (1), actual_ttx.splitlines (1)): + sys.stdout.write (line) + sys.stdout.flush () return fail_test(test, cli_args, 'ttx for expected and actual does not match.') if should_check_ots: print ("Checking output with ots-sanitize.") - if not check_ots(out_file): - return fail_test(test, cli_args, 'ots for subsetted file fails.') + if not check_ots (out_file): + return fail_test (test, cli_args, 'ots for subsetted file fails.') return 0 -def run_ttx(file): +def run_ttx (file): print ("ttx %s" % file) - cli_args = ["ttx", - "-q", - "-o-", - file] - return cmd(cli_args) + return cmd([ttx, "-q", "-o-", file]) def strip_check_sum (ttx_string): return re.sub ('checkSumAdjustment value=["]0x([0-9a-fA-F])+["]', @@ -91,14 +110,13 @@ def strip_check_sum (ttx_string): ttx_string.decode (), count=1) def has_ots (): - _, returncode = cmd(["which", "ots-sanitize"]) - if returncode: + if not ots_sanitize: print("OTS is not present, skipping all ots checks.") return False return True def check_ots (path): - ots_report, returncode = cmd(["ots-sanitize", path]) + ots_report, returncode = cmd ([ots_sanitize, path]) if returncode: print("OTS Failure: %s" % ots_report); return False @@ -110,24 +128,19 @@ if not args or sys.argv[1].find('hb-subset') == -1 or not os.path.exists (sys.ar sys.exit (1) hb_subset, args = args[0], args[1:] -if not len(args): +if not len (args): print ("No tests supplied.") sys.exit (1) -_, returncode = cmd(["which", "ttx"]) -if returncode: - print("TTX is not present, skipping test.") - sys.exit (77) - has_ots = has_ots() fails = 0 for path in args: - with io.open(path, mode="r", encoding="utf-8") as f: + with io.open (path, mode="r", encoding="utf-8") as f: print ("Running tests in " + path) - test_suite = SubsetTestSuite(path, f.read()) - for test in test_suite.tests(): - fails += run_test(test, has_ots) + test_suite = SubsetTestSuite (path, f.read()) + for test in test_suite.tests (): + fails += run_test (test, has_ots) if fails != 0: print (str (fails) + " test(s) failed.") commit 159ddb872986f121818e816d2ea75d271075ba1f Author: Ebrahim Byagowi <[email protected]> Date: Sun Jun 24 23:09:16 2018 +0430 Treat - just as /dev/stdin and remove one extra file reader (#1065) diff --git a/util/options.cc b/util/options.cc index 682e40c4..57cc4aa8 100644 --- a/util/options.cc +++ b/util/options.cc @@ -647,29 +647,20 @@ font_options_t::get_font (void) const if (!font_file) fail (true, "No font file set"); - if (0 == strcmp (font_file, "-")) { - /* read it */ - GString *gs = g_string_new (nullptr); - char buf[BUFSIZ]; + const char *font_path = font_file; + + if (0 == strcmp (font_path, "-")) + { #if defined(_WIN32) || defined(__CYGWIN__) setmode (fileno (stdin), O_BINARY); + font_path = "STDIN"; +#else + font_path = "/dev/stdin"; #endif - while (!feof (stdin)) { - size_t ret = fread (buf, 1, sizeof (buf), stdin); - if (ferror (stdin)) - fail (false, "Failed reading font from standard input: %s", - strerror (errno)); - g_string_append_len (gs, buf, ret); - } - unsigned int len = gs->len; - char *font_data = g_string_free (gs, false); - blob = hb_blob_create (font_data, len, - HB_MEMORY_MODE_WRITABLE, font_data, - (hb_destroy_func_t) g_free); - } else { - blob = hb_blob_create_from_file (font_file); } + blob = hb_blob_create_from_file (font_path); + if (blob == hb_blob_get_empty ()) fail (false, "No such file or directory"); _______________________________________________ HarfBuzz mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/harfbuzz
