diff --git a/arch/arm/cpu/armv8/start.S b/arch/arm/cpu/armv8/start.S
index 7a21560416..b5ada61062 100644
--- a/arch/arm/cpu/armv8/start.S
+++ b/arch/arm/cpu/armv8/start.S
@@ -231,7 +231,7 @@ ENDPROC(apply_core_errata)
 WEAK(lowlevel_init)
 	mov	x29, lr			/* Save LR */
 
-#if defined(CONFIG_GICV2) || defined(CONFIG_GICV3)
+#if !defined(CONFIG_SPL_BUILD) && defined(CONFIG_IRQ)
 	branch_if_slave x0, 1f
 	ldr	x0, =GICD_BASE
 	bl	gic_init_secure
@@ -246,7 +246,7 @@ WEAK(lowlevel_init)
 #endif
 #endif
 
-#if defined(CONFIG_IRQ)
+#if !defined(CONFIG_SPL_BUILD) && defined(CONFIG_IRQ)
 	/*
 	 * Setting HCR_EL2.TGE AMO IMO FMO for exception rounting to EL2
 	 */
diff --git a/arch/arm/lib/gic_64.S b/arch/arm/lib/gic_64.S
index febd967e28..1e80fb93c2 100644
--- a/arch/arm/lib/gic_64.S
+++ b/arch/arm/lib/gic_64.S
@@ -108,7 +108,7 @@ ENTRY(gic_init_secure_percpu)
 	mov	w11, #0x1		/* Enable SGI 0 */
 	str	w11, [x10, GICR_ISENABLERn]
 
-#ifdef CONFIG_IRQ
+#if !defined(CONFIG_SPL_BUILD) && defined(CONFIG_IRQ)
 	/* Rockchip: check elx */
 	switch_el x0, el3_sre, el2_sre, el1_sre
 
diff --git a/arch/arm/lib/interrupts.c b/arch/arm/lib/interrupts.c
index 6b8ec16776..69cf54bd59 100644
--- a/arch/arm/lib/interrupts.c
+++ b/arch/arm/lib/interrupts.c
@@ -26,7 +26,7 @@
 
 DECLARE_GLOBAL_DATA_PTR;
 
-#ifndef CONFIG_IRQ
+#if defined(CONFIG_SPL_BUILD) || !defined(CONFIG_IRQ)
 int interrupt_init (void)
 {
 	/*
@@ -156,7 +156,7 @@ void do_fiq (struct pt_regs *pt_regs)
 	bad_mode ();
 }
 
-#ifndef CONFIG_IRQ
+#if defined(CONFIG_SPL_BUILD) || !defined(CONFIG_IRQ)
 void do_irq (struct pt_regs *pt_regs)
 {
 	efi_restore_gd();
diff --git a/arch/arm/lib/interrupts_64.c b/arch/arm/lib/interrupts_64.c
index e151f80625..65e313e844 100644
--- a/arch/arm/lib/interrupts_64.c
+++ b/arch/arm/lib/interrupts_64.c
@@ -9,7 +9,7 @@
 #include <linux/compiler.h>
 #include <efi_loader.h>
 
-#ifndef CONFIG_IRQ
+#if defined(CONFIG_SPL_BUILD) || !defined(CONFIG_IRQ)
 int interrupt_init(void)
 {
 	return 0;
@@ -93,7 +93,7 @@ void do_sync(struct pt_regs *pt_regs, unsigned int esr)
 	panic("Resetting CPU ...\n");
 }
 
-#ifndef CONFIG_IRQ
+#if defined(CONFIG_SPL_BUILD) || !defined(CONFIG_IRQ)
 /*
  * do_irq handles the Irq exception.
  */
diff --git a/arch/arm/lib/stack.c b/arch/arm/lib/stack.c
index 5f15e34ed5..67e01b9865 100644
--- a/arch/arm/lib/stack.c
+++ b/arch/arm/lib/stack.c
@@ -25,7 +25,7 @@ int arch_reserve_stacks(void)
 	gd->irq_sp = gd->start_addr_sp;
 
 # if !defined(CONFIG_ARM64)
-#ifdef CONFIG_IRQ
+#if !defined(CONFIG_SPL_BUILD) && defined(CONFIG_IRQ)
 #ifndef CONFIG_IRQ_STACK_SIZE
 #define CONFIG_IRQ_STACK_SIZE	8192
 #endif
diff --git a/arch/arm/lib/vectors.S b/arch/arm/lib/vectors.S
index 7079641213..9896b60652 100644
--- a/arch/arm/lib/vectors.S
+++ b/arch/arm/lib/vectors.S
@@ -147,7 +147,7 @@ IRQ_STACK_START_IN:
 	.word	0x0badc0de
 #endif
 
-#ifdef CONFIG_IRQ
+#if !defined(CONFIG_SPL_BUILD) && defined(CONFIG_IRQ)
 /* IRQ stack memory (calculated at run-time) */
 .globl IRQ_STACK_START
 IRQ_STACK_START:
@@ -280,7 +280,7 @@ not_used:
 
 	.align	5
 irq:
-#ifdef CONFIG_IRQ
+#if !defined(CONFIG_SPL_BUILD) && defined(CONFIG_IRQ)
 	get_irq_stack
 	irq_save_user_regs
 	bl	do_irq
diff --git a/drivers/Makefile b/drivers/Makefile
index 9f961fd3ed..b00a2ca1ae 100644
--- a/drivers/Makefile
+++ b/drivers/Makefile
@@ -16,7 +16,7 @@ obj-$(CONFIG_$(SPL_TPL_)SERIAL_SUPPORT) += serial/
 obj-$(CONFIG_$(SPL_TPL_)SPI_FLASH_SUPPORT) += mtd/spi/
 obj-$(CONFIG_$(SPL_TPL_)SPI_SUPPORT) += spi/
 obj-$(CONFIG_$(SPL_TPL_)TIMER) += timer/
-obj-$(CONFIG_IRQ) += irq/
+obj-$(CONFIG_$(SPL_TPL_)IRQ) += irq/
 
 ifndef CONFIG_TPL_BUILD
 ifdef CONFIG_SPL_BUILD