diff --git a/kittens/choose_files/main.go b/kittens/choose_files/main.go
index 2eaf2fb05..2722f92b8 100644
--- a/kittens/choose_files/main.go
+++ b/kittens/choose_files/main.go
@@ -3,6 +3,7 @@ package choose_files
 import (
 	"fmt"
 	"os"
+	"regexp"
 	"strings"
 
 	"github.com/kovidgoyal/kitty/tools/cli"
@@ -21,19 +22,17 @@ type State struct {
 	select_dirs      bool
 	multiselect      bool
 	max_depth        int
-	exclude_patterns *utils.Set[string]
+	exclude_patterns []*regexp.Regexp
 	search_text      string
 }
 
-func (s State) BaseDir() string    { return utils.IfElse(s.base_dir == "", default_cwd, s.base_dir) }
-func (s State) SelectDirs() bool   { return s.select_dirs }
-func (s State) Multiselect() bool  { return s.multiselect }
-func (s State) MaxDepth() int      { return utils.IfElse(s.max_depth < 1, 4, s.max_depth) }
-func (s State) String() string     { return utils.Repr(s) }
-func (s State) SearchText() string { return s.search_text }
-func (s State) ExcludePatterns() *utils.Set[string] {
-	return utils.IfElse(s.exclude_patterns == nil, utils.NewSet[string](2), s.exclude_patterns)
-}
+func (s State) BaseDir() string                   { return utils.IfElse(s.base_dir == "", default_cwd, s.base_dir) }
+func (s State) SelectDirs() bool                  { return s.select_dirs }
+func (s State) Multiselect() bool                 { return s.multiselect }
+func (s State) MaxDepth() int                     { return utils.IfElse(s.max_depth < 1, 4, s.max_depth) }
+func (s State) String() string                    { return utils.Repr(s) }
+func (s State) SearchText() string                { return s.search_text }
+func (s State) ExcludePatterns() []*regexp.Regexp { return s.exclude_patterns }
 func (s State) CurrentDir() string {
 	return utils.IfElse(s.current_dir == "", s.BaseDir(), s.current_dir)
 }
@@ -112,14 +111,20 @@ func (h *Handler) OnText(text string, from_key_event, in_bracketed_paste bool) (
 
 func (h *Handler) set_state_from_config(conf *Config) (err error) {
 	h.state.max_depth = int(conf.Max_depth)
-	h.state.exclude_patterns = utils.NewSet[string](len(conf.Exclude_directory))
+	h.state.exclude_patterns = make([]*regexp.Regexp, 0, len(conf.Exclude_directory))
+	seen := map[string]*regexp.Regexp{}
 	for _, x := range conf.Exclude_directory {
 		if strings.HasPrefix(x, "!") {
-			h.state.exclude_patterns.Discard(x[1:])
-		} else {
-			h.state.exclude_patterns.Add(x)
+			delete(seen, x[1:])
+		} else if seen[x] == nil {
+			if pat, err := regexp.Compile(x); err == nil {
+				seen[x] = pat
+			} else {
+				return fmt.Errorf("The exclude directory pattern %#v is invalid: %w", x, err)
+			}
 		}
 	}
+	h.state.exclude_patterns = utils.Values(seen)
 	return
 }
 
diff --git a/kittens/choose_files/main.py b/kittens/choose_files/main.py
index 8af13738e..95e5fe6c9 100644
--- a/kittens/choose_files/main.py
+++ b/kittens/choose_files/main.py
@@ -18,13 +18,20 @@
 mma = definition.add_mouse_map
 
 agr('scan', 'Scanning the filesystem')
-opt('+exclude_directory', '/proc', add_to_default=True, long_text='''
-Pattern to exclude directories. Can be specified multiple times. Matches against the absolute path to the directory.
-If the pattern starts with :code:`!`, the :code:`!` is removed and the remaning pattern is removed from the list of patterns. This
+opt(
+    '+exclude_directory',
+    '^/proc$',
+    add_to_default=True,
+    long_text='''
+Regular expression to exclude directories. Matching directories will not be recursed into, but
+you can still or change into them to inspect their contents. Can be specified multiple times. Matches against the absolute path to the directory.
+If the pattern starts with :code:`!`, the :code:`!` is removed and the remaining pattern is removed from the list of patterns. This
 can be used to remove the default excluded directory patterns.
-''')
-opt('+exclude_directory', '/dev', add_to_default=True)
-opt('+exclude_directory', '/sys', add_to_default=True)
+''',
+)
+opt('+exclude_directory', '^/dev$', add_to_default=True)
+opt('+exclude_directory', '^/sys$', add_to_default=True)
+opt('+exclude_directory', '/__pycache__$', add_to_default=True)
 
 opt('max_depth', '4', option_type='positive_int', long_text='''
 The maximum depth to which to scan the filesystem for matches. Using large values will slow things down considerably. The better
diff --git a/kittens/choose_files/scan.go b/kittens/choose_files/scan.go
index f963550ce..292122d90 100644
--- a/kittens/choose_files/scan.go
+++ b/kittens/choose_files/scan.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"regexp"
 	"slices"
 	"strings"
 	"sync"
@@ -64,16 +65,16 @@ func scan_dir(path, root_dir string) []ResultItem {
 	return []ResultItem{}
 }
 
-func is_excluded(path string, exclude_patterns *utils.Set[string]) bool {
-	for pattern := range exclude_patterns.Iterable() {
-		if matched, _ := filepath.Match(pattern, path); matched {
+func is_excluded(path string, exclude_patterns []*regexp.Regexp) bool {
+	for _, pattern := range exclude_patterns {
+		if pattern.MatchString(path) {
 			return true
 		}
 	}
 	return false
 }
 
-func (sc *ScanCache) fs_scan(root_dir, current_dir string, max_depth int, exclude_patterns *utils.Set[string], seen map[string]bool) (ans []ResultItem) {
+func (sc *ScanCache) fs_scan(root_dir, current_dir string, max_depth int, exclude_patterns []*regexp.Regexp, seen map[string]bool) (ans []ResultItem) {
 	var found bool
 	if ans, found = sc.get_cached_entries(current_dir); !found {
 		ans = scan_dir(current_dir, root_dir)
@@ -92,7 +93,7 @@ func (sc *ScanCache) fs_scan(root_dir, current_dir string, max_depth int, exclud
 	return
 }
 
-func (sc *ScanCache) scan(root_dir, search_text string, max_depth int, exclude_patterns *utils.Set[string]) (ans []ResultItem) {
+func (sc *ScanCache) scan(root_dir, search_text string, max_depth int, exclude_patterns []*regexp.Regexp) (ans []ResultItem) {
 	seen := make(map[string]bool, 1024)
 	ans = sc.fs_scan(root_dir, root_dir, max_depth, exclude_patterns, seen)
 	if search_text == "" {