Rework Fontconfig fallback to use cached list from font_sort

Previous implementation was querying Fontconfig using `charset` in a pattern,
which was leading to unpredictable fallbacks in some cases, since Fontconfig
was picking the font with the most coverage for a given charset, regardless of
user configuration. Moreover all fallback was based on font_match which is
extremely slow for such performance sensitive task as a fallback, so alacritty
had a hard times on vtebench's unicode-random-write.

The new approach is to use some internal fallback list from font_sort
and iterate over it to get a proper fallback font, since it matches the
following example query from `fc-match`:

`fc-match -s "monospace:pixelsize=X:style=Y"

That being said it's more intuitive for users to setup their system Fontconfig
fallback, and also most applications are doing similar things. Moreover the new
implementation uses internal caches over Fontconfig API when possible and
performs font matches only once during load of requested font with font_sort,
which leads to dramatically improved performance on already mentioned
vtebench's unicode-random-write.

Fixes #3176.
Fixes #3134.
Fixes #2657.
Fixes #1560.
Fixes #965.
Fixes #511.

CHANGELOG.md

@@ -38,6 +38,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Underline position for bitmap fonts
 - Selection rotating outside of scrolling region
 - Throughput performance problems caused by excessive font metric queries
+- Unicode throughput performance on Linux/BSD
+- Resize of bitmap fonts
+- Crash when using bitmap font with `embeddedbitmap` set to `false`
+- Inconsistent fontconfig fallback
 
 ### Removed
 

font/src/ft/fc/char_set.rs

@@ -13,15 +13,19 @@
 // limitations under the License.
 use std::ptr::NonNull;
 
-use foreign_types::{foreign_type, ForeignTypeRef};
+use foreign_types::{foreign_type, ForeignType, ForeignTypeRef};
 
 use super::ffi::FcCharSetCreate;
-use super::ffi::{FcCharSet, FcCharSetAddChar, FcCharSetDestroy};
+use super::ffi::{
+    FcBool, FcCharSet, FcCharSetAddChar, FcCharSetCopy, FcCharSetCount, FcCharSetDestroy,
+    FcCharSetHasChar, FcCharSetMerge, FcCharSetSubtract, FcCharSetUnion,
+};
 
 foreign_type! {
     pub unsafe type CharSet {
         type CType = FcCharSet;
         fn drop = FcCharSetDestroy;
+        fn clone = FcCharSetCopy;
     }
 }
 
@@ -41,4 +45,39 @@ impl CharSetRef {
     pub fn add(&mut self, glyph: char) -> bool {
         unsafe { FcCharSetAddChar(self.as_ptr(), glyph as _) == 1 }
     }
+
+    pub fn has_char(&self, glyph: char) -> bool {
+        unsafe { FcCharSetHasChar(self.as_ptr(), glyph as _) == 1 }
+    }
+
+    pub fn count(&self) -> u32 {
+        unsafe { FcCharSetCount(self.as_ptr()) as u32 }
+    }
+
+    pub fn union(&self, other: &CharSetRef) -> CharSet {
+        unsafe {
+            let ptr = FcCharSetUnion(self.as_ptr() as _, other.as_ptr() as _);
+            CharSet::from_ptr(ptr)
+        }
+    }
+
+    pub fn subtract(&self, other: &CharSetRef) -> CharSet {
+        unsafe {
+            let ptr = FcCharSetSubtract(self.as_ptr() as _, other.as_ptr() as _);
+            CharSet::from_ptr(ptr)
+        }
+    }
+
+    pub fn merge(&self, other: &CharSetRef) -> Result<bool, ()> {
+        unsafe {
+            // Value is just an indicator whether something was added or not
+            let mut value: FcBool = 0;
+            let res = FcCharSetMerge(self.as_ptr() as _, other.as_ptr() as _, &mut value);
+            if res == 0 {
+                Err(())
+            } else {
+                Ok(value != 0)
+            }
+        }
+    }
 }

font/src/ft/fc/mod.rs

@@ -75,11 +75,10 @@ pub fn font_sort(config: &ConfigRef, pattern: &mut PatternRef) -> Option<FontSet
         let mut result = FcResultNoMatch;
 
         let mut charsets: *mut _ = ptr::null_mut();
-
         let ptr = FcFontSort(
             config.as_ptr(),
             pattern.as_ptr(),
-            0, // false
+            1, // Trim font list
             &mut charsets,
             &mut result,
         );
@@ -323,7 +322,7 @@ mod tests {
         let fonts = super::font_sort(config, &mut pattern).expect("sort font monospace");
 
         for font in fonts.into_iter().take(10) {
-            let font = font.render_prepare(&config, &pattern);
+            let font = pattern.render_prepare(&config, &font);
             print!("index={:?}; ", font.index());
             print!("family={:?}; ", font.family());
             print!("style={:?}; ", font.style());
@@ -345,7 +344,7 @@ mod tests {
         let fonts = super::font_sort(config, &mut pattern).expect("font_sort");
 
         for font in fonts.into_iter().take(10) {
-            let font = font.render_prepare(&config, &pattern);
+            let font = pattern.render_prepare(&config, &font);
             print!("index={:?}; ", font.index());
             print!("family={:?}; ", font.family());
             print!("style={:?}; ", font.style());

font/src/ft/fc/pattern.rs

@@ -24,7 +24,7 @@ use libc::{c_char, c_double, c_int};
 use super::ffi::FcResultMatch;
 use super::ffi::{FcBool, FcFontRenderPrepare, FcPatternGetBool, FcPatternGetDouble};
 use super::ffi::{FcChar8, FcConfigSubstitute, FcDefaultSubstitute, FcPattern};
-use super::ffi::{FcPatternAddCharSet, FcPatternDestroy};
+use super::ffi::{FcPatternAddCharSet, FcPatternDestroy, FcPatternDuplicate, FcPatternGetCharSet};
 use super::ffi::{FcPatternAddDouble, FcPatternAddString, FcPatternCreate, FcPatternGetString};
 use super::ffi::{FcPatternAddInteger, FcPatternGetInteger, FcPatternPrint};
 
@@ -329,6 +329,7 @@ foreign_type! {
     pub unsafe type Pattern {
         type CType = FcPattern;
         fn drop = FcPatternDestroy;
+        fn clone = FcPatternDuplicate;
     }
 }
 
@@ -531,7 +532,7 @@ impl PatternRef {
 
     pub fn render_prepare(&self, config: &ConfigRef, request: &PatternRef) -> Pattern {
         unsafe {
-            let ptr = FcFontRenderPrepare(config.as_ptr(), request.as_ptr(), self.as_ptr());
+            let ptr = FcFontRenderPrepare(config.as_ptr(), self.as_ptr(), request.as_ptr());
             Pattern::from_ptr(ptr)
         }
     }
@@ -552,6 +553,25 @@ impl PatternRef {
         }
     }
 
+    pub fn get_charset(&self) -> Option<&CharSetRef> {
+        unsafe {
+            let mut charset: *mut _ = ptr::null_mut();
+
+            let result = FcPatternGetCharSet(
+                self.as_ptr(),
+                b"charset\0".as_ptr() as *mut c_char,
+                0,
+                &mut charset,
+            );
+
+            if result == FcResultMatch {
+                Some(&*(charset as *const CharSetRef))
+            } else {
+                None
+            }
+        }
+    }
+
     pub fn file(&self, index: usize) -> Option<PathBuf> {
         unsafe { self.get_string(b"file\0").nth(index) }.map(From::from)
     }

font/src/ft/mod.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 //
-//! Rasterization powered by FreeType and FontConfig
+//! Rasterization powered by FreeType and Fontconfig.
 use std::cmp::{min, Ordering};
 use std::collections::HashMap;
 use std::fmt::{self, Display, Formatter};
@@ -26,6 +26,8 @@ use log::{debug, trace};
 
 pub mod fc;
 
+use fc::{CharSet, Pattern, PatternRef};
+
 use super::{
     BitmapBuffer, FontDesc, FontKey, GlyphKey, Metrics, Rasterize, RasterizedGlyph, Size, Slant,
     Style, Weight,
@@ -35,6 +37,12 @@ struct FixedSize {
     pixelsize: f64,
 }
 
+#[derive(Default)]
+struct FallbackList {
+    list: Vec<Pattern>,
+    coverage: CharSet,
+}
+
 struct Face {
     ft_face: freetype::Face,
     key: FontKey,
@@ -70,6 +78,7 @@ pub struct FreeTypeRasterizer {
     faces: HashMap<FontKey, Face>,
     library: Library,
     keys: HashMap<PathBuf, FontKey>,
+    fallback_lists: HashMap<FontKey, FallbackList>,
     device_pixel_ratio: f32,
     pixel_size: f64,
 }
@@ -88,6 +97,7 @@ impl Rasterize for FreeTypeRasterizer {
         Ok(FreeTypeRasterizer {
             faces: HashMap::new(),
             keys: HashMap::new(),
+            fallback_lists: HashMap::new(),
             library,
             device_pixel_ratio,
             pixel_size: 0.0,
@@ -223,34 +233,88 @@ impl FreeTypeRasterizer {
         slant: Slant,
         weight: Weight,
     ) -> Result<FontKey, Error> {
-        let mut pattern = fc::Pattern::new();
+        let mut pattern = Pattern::new();
         pattern.add_family(&desc.name);
         pattern.set_weight(weight.into_fontconfig_type());
         pattern.set_slant(slant.into_fontconfig_type());
         pattern.add_pixelsize(self.pixel_size);
 
-        let font = fc::font_match(fc::Config::get_current(), &mut pattern)
+        self.query_font(pattern, desc)
-            .ok_or_else(|| Error::MissingFont(desc.to_owned()))?;
-
-        self.face_from_pattern(&font).and_then(|pattern| {
-            pattern.map(Ok).unwrap_or_else(|| Err(Error::MissingFont(desc.to_owned())))
-        })
     }
 
     fn get_specific_face(&mut self, desc: &FontDesc, style: &str) -> Result<FontKey, Error> {
-        let mut pattern = fc::Pattern::new();
+        let mut pattern = Pattern::new();
         pattern.add_family(&desc.name);
         pattern.add_style(style);
         pattern.add_pixelsize(self.pixel_size);
 
-        let font = fc::font_match(fc::Config::get_current(), &mut pattern)
+        self.query_font(pattern, desc)
-            .ok_or_else(|| Error::MissingFont(desc.to_owned()))?;
-        self.face_from_pattern(&font).and_then(|pattern| {
-            pattern.map(Ok).unwrap_or_else(|| Err(Error::MissingFont(desc.to_owned())))
-        })
     }
 
-    fn face_from_pattern(&mut self, pattern: &fc::Pattern) -> Result<Option<FontKey>, Error> {
+    fn query_font(&mut self, pattern: Pattern, desc: &FontDesc) -> Result<FontKey, Error> {
+        let config = fc::Config::get_current();
+        let fonts = fc::font_sort(&config, &mut pattern.clone())
+            .ok_or_else(|| Error::MissingFont(desc.to_owned()))?;
+
+        let mut font_iter = fonts.into_iter();
+
+        let base_font = font_iter.next().ok_or_else(|| Error::MissingFont(desc.to_owned()))?;
+        let base_font = pattern.render_prepare(config, base_font);
+
+        let font_path = base_font.file(0).ok_or_else(|| Error::MissingFont(desc.to_owned()))?;
+
+        // Reload already loaded faces and drop their fallback faces
+        let font_key = if let Some(font_key) = self.keys.remove(&font_path) {
+            let fallback_list = self.fallback_lists.remove(&font_key).unwrap_or_default();
+
+            for font_pattern in &fallback_list.list {
+                let path = match font_pattern.file(0) {
+                    Some(path) => path,
+                    None => continue,
+                };
+
+                if let Some(ff_key) = self.keys.get(&path) {
+                    // Skip primary fonts, since these are all reloaded later
+                    if !self.fallback_lists.contains_key(&ff_key) {
+                        self.faces.remove(ff_key);
+                        self.keys.remove(&path);
+                    }
+                }
+            }
+
+            let _ = self.faces.remove(&font_key);
+            Some(font_key)
+        } else {
+            None
+        };
+
+        // Reuse the font_key, since changing it can break library users
+        let font_key = self.face_from_pattern(&base_font, font_key).and_then(|pattern| {
+            pattern.map(Ok).unwrap_or_else(|| Err(Error::MissingFont(desc.to_owned())))
+        })?;
+
+        // Coverage for fallback fonts
+        let coverage = CharSet::new();
+        let empty_charset = CharSet::new();
+        // Load fallback list
+        let list: Vec<Pattern> = font_iter
+            .map(|font| {
+                let charset = font.get_charset().unwrap_or(&empty_charset);
+                let _ = coverage.merge(&charset);
+                font.to_owned()
+            })
+            .collect();
+
+        self.fallback_lists.insert(font_key, FallbackList { list, coverage });
+
+        Ok(font_key)
+    }
+
+    fn face_from_pattern(
+        &mut self,
+        pattern: &PatternRef,
+        key: Option<FontKey>,
+    ) -> Result<Option<FontKey>, Error> {
         if let (Some(path), Some(index)) = (pattern.file(0), pattern.index().next()) {
             if let Some(key) = self.keys.get(&path) {
                 return Ok(Some(*key));
@@ -279,9 +343,12 @@ impl FreeTypeRasterizer {
                 }
             }
 
+            // Reuse the original fontkey if you're reloading the font
+            let key = if let Some(key) = key { key } else { FontKey::next() };
+
             let face = Face {
                 ft_face,
-                key: FontKey::next(),
+                key,
                 load_flags: Self::ft_load_flags(pattern),
                 render_mode: Self::ft_render_mode(pattern),
                 lcd_filter: Self::ft_lcd_filter(pattern),
@@ -295,7 +362,6 @@ impl FreeTypeRasterizer {
             let key = face.key;
             self.faces.insert(key, face);
             self.keys.insert(path, key);
-
             Ok(Some(key))
         } else {
             Ok(None)
@@ -320,7 +386,7 @@ impl FreeTypeRasterizer {
         if use_initial_face {
             Ok(glyph_key.font_key)
         } else {
-            let key = self.load_face_with_glyph(c).unwrap_or(glyph_key.font_key);
+            let key = self.load_face_with_glyph(glyph_key).unwrap_or(glyph_key.font_key);
             Ok(key)
         }
     }
@@ -375,12 +441,13 @@ impl FreeTypeRasterizer {
         }
     }
 
-    fn ft_load_flags(pat: &fc::Pattern) -> freetype::face::LoadFlag {
+    fn ft_load_flags(pattern: &PatternRef) -> freetype::face::LoadFlag {
-        let antialias = pat.antialias().next().unwrap_or(true);
+        let antialias = pattern.antialias().next().unwrap_or(true);
-        let hinting = pat.hintstyle().next().unwrap_or(fc::HintStyle::Slight);
+        let hinting = pattern.hintstyle().next().unwrap_or(fc::HintStyle::Slight);
-        let rgba = pat.rgba().next().unwrap_or(fc::Rgba::Unknown);
+        let rgba = pattern.rgba().next().unwrap_or(fc::Rgba::Unknown);
-        let embedded_bitmaps = pat.embeddedbitmap().next().unwrap_or(true);
+        let embedded_bitmaps = pattern.embeddedbitmap().next().unwrap_or(true);
-        let color = pat.color().next().unwrap_or(false);
+        let scalable = pattern.scalable().next().unwrap_or(true);
+        let color = pattern.color().next().unwrap_or(false);
 
         use freetype::face::LoadFlag;
         let mut flags = match (antialias, hinting, rgba) {
@@ -414,7 +481,9 @@ impl FreeTypeRasterizer {
             (true, _, fc::Rgba::None) => LoadFlag::TARGET_NORMAL,
         };
 
-        if !embedded_bitmaps {
+        // Non scalable fonts only have bitmaps, so disabling them entirely is likely not a
+        // desirable thing. Colored fonts aren't scalable, but also only have bitmaps.
+        if !embedded_bitmaps && scalable && !color {
             flags |= LoadFlag::NO_BITMAP;
         }
 
@@ -425,7 +494,7 @@ impl FreeTypeRasterizer {
         flags
     }
 
-    fn ft_render_mode(pat: &fc::Pattern) -> freetype::RenderMode {
+    fn ft_render_mode(pat: &PatternRef) -> freetype::RenderMode {
         let antialias = pat.antialias().next().unwrap_or(true);
         let rgba = pat.rgba().next().unwrap_or(fc::Rgba::Unknown);
 
@@ -437,7 +506,7 @@ impl FreeTypeRasterizer {
         }
     }
 
-    fn ft_lcd_filter(pat: &fc::Pattern) -> c_uint {
+    fn ft_lcd_filter(pat: &PatternRef) -> c_uint {
         match pat.lcdfilter().next().unwrap_or(fc::LcdFilter::Default) {
             fc::LcdFilter::None => freetype::ffi::FT_LCD_FILTER_NONE,
             fc::LcdFilter::Default => freetype::ffi::FT_LCD_FILTER_DEFAULT,
@@ -537,48 +606,57 @@ impl FreeTypeRasterizer {
         }
     }
 
-    fn load_face_with_glyph(&mut self, glyph: char) -> Result<FontKey, Error> {
+    fn load_face_with_glyph(&mut self, glyph: GlyphKey) -> Result<FontKey, Error> {
-        let mut charset = fc::CharSet::new();
+        let fallback_list = self.fallback_lists.get(&glyph.font_key).unwrap();
-        charset.add(glyph);
-        let mut pattern = fc::Pattern::new();
-        pattern.add_charset(&charset);
-        pattern.add_pixelsize(self.pixel_size as f64);
 
-        let config = fc::Config::get_current();
+        // Check whether glyph is presented in any fallback font
-        match fc::font_match(config, &mut pattern) {
+        if !fallback_list.coverage.has_char(glyph.c) {
-            Some(pattern) => {
+            return Ok(glyph.font_key);
-                if let (Some(path), Some(_)) = (pattern.file(0), pattern.index().next()) {
-                    match self.keys.get(&path) {
-                        // We've previously loaded this font, so don't
-                        // load it again.
-                        Some(&key) => {
-                            debug!("Hit for font {:?}; no need to load", path);
-                            // Update fixup factor
-                            self.faces.get_mut(&key).unwrap().pixelsize_fixup_factor =
-                                pattern.pixelsizefixupfactor().next();
-                            Ok(key)
-                        },
-
-                        None => {
-                            debug!("Miss for font {:?}; loading now", path);
-                            // Safe to unwrap the option since we've already checked for the path
-                            // and index above.
-                            let key = self.face_from_pattern(&pattern)?.unwrap();
-                            Ok(key)
-                        },
-                    }
-                } else {
-                    Err(Error::MissingFont(FontDesc::new(
-                        "fallback-without-path",
-                        Style::Specific(glyph.to_string()),
-                    )))
-                }
-            },
-            None => Err(Error::MissingFont(FontDesc::new(
-                "no-fallback-for",
-                Style::Specific(glyph.to_string()),
-            ))),
         }
+
+        for font_pattern in &fallback_list.list {
+            let path = match font_pattern.file(0) {
+                Some(path) => path,
+                None => continue,
+            };
+
+            match self.keys.get(&path) {
+                Some(&key) => {
+                    let face = match self.faces.get(&key) {
+                        Some(face) => face,
+                        None => continue,
+                    };
+
+                    let index = face.ft_face.get_char_index(glyph.c as usize);
+
+                    // We found something in a current face, so let's use it
+                    if index != 0 {
+                        return Ok(key);
+                    }
+                },
+                None => {
+                    if font_pattern.get_charset().map(|cs| cs.has_char(glyph.c)) != Some(true) {
+                        continue;
+                    }
+
+                    // Recreate a pattern
+                    let mut pattern = Pattern::new();
+                    pattern.add_pixelsize(self.pixel_size as f64);
+                    pattern.add_style(font_pattern.style().next().unwrap_or("Regular"));
+                    pattern.add_family(font_pattern.family().next().unwrap_or("monospace"));
+
+                    // Render pattern, otherwise most of its properties wont work
+                    let config = fc::Config::get_current();
+                    let pattern = pattern.render_prepare(config, font_pattern);
+
+                    let key = self.face_from_pattern(&pattern, None)?.unwrap();
+                    return Ok(key);
+                },
+            }
+        }
+
+        // You can hit this return, if you're failing to get charset from a pattern
+        Ok(glyph.font_key)
     }
 }