diff --git a/src/notification.c b/src/notification.c
index 66d9b3b..9657cc1 100644
--- a/src/notification.c
+++ b/src/notification.c
@@ -22,7 +22,6 @@
 #include "rules.h"
 #include "settings.h"
 #include "utils.h"
-#include "x11/x.h"
 
 static void notification_extract_urls(struct notification *n);
 static void notification_format_message(struct notification *n);
@@ -348,12 +347,26 @@ void notification_init(struct notification *n)
                 n->icon = g_strdup(settings.icons[n->urgency]);
 
         /* Color hints */
+        struct notification_colors defcolors;
+        switch (n->urgency) {
+                case URG_LOW:
+                        defcolors = settings.colors_low;
+                        break;
+                case URG_NORM:
+                        defcolors = settings.colors_norm;
+                        break;
+                case URG_CRIT:
+                        defcolors = settings.colors_crit;
+                        break;
+                default:
+                        g_error("Unhandled urgency type: %d", n->urgency);
+        }
         if (!n->colors.fg)
-                n->colors.fg = g_strdup(xctx.colors[ColFG][n->urgency]);
+                n->colors.fg = g_strdup(defcolors.fg);
         if (!n->colors.bg)
-                n->colors.bg = g_strdup(xctx.colors[ColBG][n->urgency]);
+                n->colors.bg = g_strdup(defcolors.bg);
         if (!n->colors.frame)
-                n->colors.frame = g_strdup(xctx.colors[ColFrame][n->urgency]);
+                n->colors.frame = g_strdup(defcolors.frame);
 
         /* Sanitize misc hints */
         if (n->progress < 0)
diff --git a/src/settings.c b/src/settings.c
index fb04d85..64883d2 100644
--- a/src/settings.c
+++ b/src/settings.c
@@ -635,7 +635,7 @@ void load_settings(char *cmdline_config_path)
 
         settings.colors_low.frame = option_get_string(
                 "urgency_low",
-                "frame_color", "-lfr", NULL,
+                "frame_color", "-lfr", settings.frame_color ? settings.frame_color : defaults.colors_low.frame,
                 "Frame color for notifications with low urgency"
         );
 
@@ -665,7 +665,7 @@ void load_settings(char *cmdline_config_path)
 
         settings.colors_norm.frame = option_get_string(
                 "urgency_normal",
-                "frame_color", "-nfr", NULL,
+                "frame_color", "-nfr", settings.frame_color ? settings.frame_color : defaults.colors_norm.frame,
                 "Frame color for notifications with normal urgency"
         );
 
@@ -695,7 +695,7 @@ void load_settings(char *cmdline_config_path)
 
         settings.colors_crit.frame = option_get_string(
                 "urgency_critical",
-                "frame_color", "-cfr", NULL,
+                "frame_color", "-cfr", settings.frame_color ? settings.frame_color : defaults.colors_crit.frame,
                 "Frame color for notifications with critical urgency"
         );
 
diff --git a/src/x11/x.c b/src/x11/x.c
index 6007fc1..8a8a285 100644
--- a/src/x11/x.c
+++ b/src/x11/x.c
@@ -464,27 +464,6 @@ void x_setup(void)
         x_shortcut_grab(&settings.context_ks);
         x_shortcut_ungrab(&settings.context_ks);
 
-        xctx.colors[ColFG][URG_LOW] = settings.colors_low.fg;
-        xctx.colors[ColFG][URG_NORM] = settings.colors_norm.fg;
-        xctx.colors[ColFG][URG_CRIT] = settings.colors_crit.fg;
-
-        xctx.colors[ColBG][URG_LOW] = settings.colors_low.bg;
-        xctx.colors[ColBG][URG_NORM] = settings.colors_norm.bg;
-        xctx.colors[ColBG][URG_CRIT] = settings.colors_crit.bg;
-
-        if (settings.colors_low.frame)
-                xctx.colors[ColFrame][URG_LOW] = settings.colors_low.frame;
-        else
-                xctx.colors[ColFrame][URG_LOW] = settings.frame_color;
-        if (settings.colors_norm.frame)
-                xctx.colors[ColFrame][URG_NORM] = settings.colors_norm.frame;
-        else
-                xctx.colors[ColFrame][URG_NORM] = settings.frame_color;
-        if (settings.colors_crit.frame)
-                xctx.colors[ColFrame][URG_CRIT] = settings.colors_crit.frame;
-        else
-                xctx.colors[ColFrame][URG_CRIT] = settings.frame_color;
-
         xctx.screensaver_info = XScreenSaverAllocInfo();
 
         init_screens();
diff --git a/src/x11/x.h b/src/x11/x.h
index 6f4ced1..3ed21e3 100644
--- a/src/x11/x.h
+++ b/src/x11/x.h
@@ -37,7 +37,6 @@ struct dimensions {
 
 struct x_context {
         Display *dpy;
-        const char *colors[3][3];
         XScreenSaverInfo *screensaver_info;
 };