3

I am learning the ARM architecture via bare metal programming on the Raspberry Pi 3. I have a booting "kernel" that prints out some info. I have reached a point that I don't understand what is happening. I have two pieces of boot code below, one works and one doesn't. I don't understand the difference. My gut says there is some sort of alignment thing I've missed in the documentation, but I can't put my finger on it.

I'm PXE booting my Pi 3, and have kernel_old=1 set in the config.txt so that it runs cores from 0x0, if this matters.

This boot.S works and I get into kernel_main and the other cores get into aux_core_main:

// File: boot.S
// Platform: Raspberry Pi 3
// Date: 05/20/2018
//
//
//
// To keep this in the first portion of the binary.
.section ".text.boot"

#define __BOOT_CODE__
#include "identifiers.h"

// GLOBALS



// Make _start global.
.globl _start


_start:
///
// all cores setup code
//
// setup the VM midr/mipdr
    mrs x0, mpidr_el1                       // Read MIPDR
    mrs x1, midr_el1                        // Read MIDR
    msr vpidr_el2, x1                       // set the VM versions of these
    msr vmpidr_el2, x0                      // same


// start core specific code
    and x0, x0, #0x03                       // mask off cpu ID bits
    cbz x0, _core0_start                    // start cpu0
    cmp x0, #0x1                            // compare and branch
    beq _core1_start
    cmp x0, #0x2                            // compare and branch
    beq _core2_start
    b   _core3_start                        // last core, just go there


_core0_start:
// get out of EL3 and into EL2
//
    mrs     x0, CurrentEL                   // read current EL
    and     x0, x0, #0x0C                   // clear reserved bits
    cmp     x0, 0x0C                        // are the two bits set (0b11 = EL3)
    bne     _exit_el3

// chaging from el3 to el2
    mov     x2, #0x5b1                      // Secure Configuration Register
    msr     scr_el3, x2                     //
    mov     x2, #0x3c9
    msr     spsr_el3, x2                    // Program Status Registers
    adr     x2, _exit_el3
    msr     elr_el3, x2
    eret                                    // go!
// So we're now in EL2, proceed with setup
_exit_el3:
    mov x0, #0
    mov x1, #ARCH_AARCH64
    mov x2, #PLATFORM_RASPI3
    ldr x3, =__dtb_start
    ldr x4, =__core_0_el2_stack_end
    mov sp, x4
    ldr x4, =__core_0_el1_stack_end
    msr sp_el1, x4
    ldr x4, =__core_0_el0_stack_end
    msr sp_el0, x4
    bl  kernel_main
    b __halt__

_core1_start:
    ldr x4, =__core_1_el2_stack_end
    mov sp, x4
    ldr x4, =__core_1_el1_stack_end
    msr sp_el1, x4
    ldr x4, =__core_1_el0_stack_end
    msr sp_el0, x4
    bl aux_core_main
    b __halt__

_core2_start:
    ldr x4, =__core_2_el2_stack_end
    mov sp, x4
    ldr x4, =__core_2_el1_stack_end
    msr sp_el1, x4
    ldr x4, =__core_2_el0_stack_end
    msr sp_el0, x4
    bl aux_core_main
    b __halt__

_core3_start:
    ldr x4, =__core_3_el2_stack_end
    mov sp, x4
    ldr x4, =__core_3_el1_stack_end
    msr sp_el1, x4
    ldr x4, =__core_3_el0_stack_end
    msr sp_el0, x4
    bl aux_core_main
    b __halt__




.globl  __halt__
__halt__:
    wfe
    b __halt__

While adding a nop before the very first instruction breaks the whole thing:

// File: boot.S
// Platform: Raspberry Pi 3
// Date: 05/20/2018
//
//
//
// To keep this in the first portion of the binary.
.section ".text.boot"

#define __BOOT_CODE__
#include "identifiers.h"

// GLOBALS



// Make _start global.
.globl _start


_start:
///
// all cores setup code
//
// setup the VM midr/mipdr
    nop
    mrs x0, mpidr_el1                       // Read MIPDR
    mrs x1, midr_el1                        // Read MIDR
    msr vpidr_el2, x1                       // set the VM versions of these
    msr vmpidr_el2, x0                      // same


// start core specific code
    and x0, x0, #0x03                       // mask off cpu ID bits
    cbz x0, _core0_start                    // start cpu0
    cmp x0, #0x1                            // compare and branch
    beq _core1_start
    cmp x0, #0x2                            // compare and branch
    beq _core2_start
    b   _core3_start                        // last core, just go there


_core0_start:
// get out of EL3 and into EL2
//
    mrs     x0, CurrentEL                   // read current EL
    and     x0, x0, #0x0C                   // clear reserved bits
    cmp     x0, 0x0C                        // are the two bits set (0b11 = EL3)
    bne     _exit_el3

// chaging from el3 to el2
    mov     x2, #0x5b1                      // Secure Configuration Register
    msr     scr_el3, x2                     //
    mov     x2, #0x3c9
    msr     spsr_el3, x2                    // Program Status Registers
    adr     x2, _exit_el3
    msr     elr_el3, x2
    eret                                    // go!
// So we're now in EL2, proceed with setup
_exit_el3:
    mov x0, #0
    mov x1, #ARCH_AARCH64
    mov x2, #PLATFORM_RASPI3
    ldr x3, =__dtb_start
    ldr x4, =__core_0_el2_stack_end
    mov sp, x4
    ldr x4, =__core_0_el1_stack_end
    msr sp_el1, x4
    ldr x4, =__core_0_el0_stack_end
    msr sp_el0, x4
    bl  kernel_main
    b __halt__

_core1_start:
    ldr x4, =__core_1_el2_stack_end
    mov sp, x4
    ldr x4, =__core_1_el1_stack_end
    msr sp_el1, x4
    ldr x4, =__core_1_el0_stack_end
    msr sp_el0, x4
    bl aux_core_main
    b __halt__

_core2_start:
    ldr x4, =__core_2_el2_stack_end
    mov sp, x4
    ldr x4, =__core_2_el1_stack_end
    msr sp_el1, x4
    ldr x4, =__core_2_el0_stack_end
    msr sp_el0, x4
    bl aux_core_main
    b __halt__

_core3_start:
    ldr x4, =__core_3_el2_stack_end
    mov sp, x4
    ldr x4, =__core_3_el1_stack_end
    msr sp_el1, x4
    ldr x4, =__core_3_el0_stack_end
    msr sp_el0, x4
    bl aux_core_main
    b __halt__




.globl  __halt__
__halt__:
    wfe
    b __halt__
Improve this question
4
  • Did you check the binary on the SD card? Funny enough, most times I encounter such mysterious errors, I'm simply using the wrong file. – Janka Jul 21 '18 at 23:33
  • I'm PXE booting, so no SD card is involved. The bootcode.bin, config.txt, start.elf, and kernel8.img are all pulled over TFTP. – Jacob Calvert Jul 22 '18 at 4:48
  • 1
    if you think that it may be an alignment problem, then try 2 or 4 NOPs – jsotola Jul 23 '18 at 7:29
  • @jsotola, I tried that too, to no avail. I'm working to pare it down to a minimal example that exhibits the issue. – Jacob Calvert Jul 26 '18 at 4:46
1

The solution lies in the config.txt.

After reading many forums and the RPi site, there is a config option called disable_commandline_tags.

If it is not set or is set to 0 in config.txt, ATAGS is loaded by start.elf starting at 0x100. My kernel was also using kernel_old=1, which starts loading at 0x0.

The ATAGs entry was overwriting a piece of the kernel. Setting disable_commandline_tags=1, no ATAGs are populated, and the problem is solved.

Improve this answer
2
  • So, not connected to the number of NOPs? – Janka Aug 4 '18 at 23:52
  • Not connected to number of NOPs. My assumption is that some critical code was around the 256B mark and the ATAGs clobbered it, the NOP added enough padding to move the code so that the ATAGs clobber it, removing it was happy coincidence. – Jacob Calvert Aug 5 '18 at 1:47

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.