Makefile file format not recognized when using cross compiler

Question

I setup a container to cross compile my program so taht i can run it on a Raspberry Pi. It compiles fine when using ${CC} but prints:

/usr/bin/arm-linux-gnueabihf-gcc -o test main.o CAcquireData.o CController.o CDriver.o CFactory.o CSensorFactory.o CDriverFactory.o CSensor.o CimuMEMS.o CimuPC.o CtmpsenTilt.o CMotorTilt.o -lstdc++ -lspidev-lib++ -lpthread -Wall -std=c++11 -I ./include
main.o: file not recognized: file format not recognized
collect2: error: ld returned 1 exit status
make: *** [Makefile:18: test] Error 1\

When the cross compiler is invoked. My Makefile looks like this:

NAME=test
IDIR=./include
CFLAGS=-lstdc++ -lspidev-lib++ -lpthread -Wall -std=c++11 -I $(IDIR)
OBJ=main.o \
    CAcquireData.o \
    CController.o \
    CDriver.o \
    CFactory.o \
    CSensorFactory.o \
    CDriverFactory.o \
    CSensor.o \
    CimuMEMS.o \
    CimuPC.o \
    CtmpsenTilt.o \
    CMotorTilt.o

test: $(OBJ)
    $(CC) -o $(NAME) $(OBJ) $(CFLAGS)

#arm: CC=/opt/pi/tools/arm-bcm2708/arm-rpi-4.9.3-linux-gnueabihf/bin/arm-linux-gnueabihf-gcc
arm: CC=/usr/bin/arm-linux-gnueabihf-gcc
arm: test 

clean:
    rm -f $(NAME) *.o

print:
    echo $(CC)
    echo $(CFLAGS)

A sequence with calling make clean in between looks like this:

# make clean
rm -f test *.o
# make
g++    -c -o main.o main.cpp
g++    -c -o CAcquireData.o CAcquireData.cpp
g++    -c -o CController.o CController.cpp
g++    -c -o CDriver.o CDriver.cpp
g++    -c -o CFactory.o CFactory.cpp
g++    -c -o CSensorFactory.o CSensorFactory.cpp
g++    -c -o CDriverFactory.o CDriverFactory.cpp
g++    -c -o CSensor.o CSensor.cpp
g++    -c -o CimuMEMS.o CimuMEMS.cpp
g++    -c -o CimuPC.o CimuPC.cpp
g++    -c -o CtmpsenTilt.o CtmpsenTilt.cpp
g++    -c -o CMotorTilt.o CMotorTilt.cpp
cc -o test main.o CAcquireData.o CController.o CDriver.o CFactory.o CSensorFactory.o CDriverFactory.o CSensor.o CimuMEMS.o CimuPC.o CtmpsenTilt.o CMotorTilt.o -lstdc++ -lspidev-lib++ -lpthread -Wall -std=c++11 -I ./include
# make clean
rm -f test *.o
# make arm
g++    -c -o main.o main.cpp
g++    -c -o CAcquireData.o CAcquireData.cpp
g++    -c -o CController.o CController.cpp
g++    -c -o CDriver.o CDriver.cpp
g++    -c -o CFactory.o CFactory.cpp
g++    -c -o CSensorFactory.o CSensorFactory.cpp
g++    -c -o CDriverFactory.o CDriverFactory.cpp
g++    -c -o CSensor.o CSensor.cpp
g++    -c -o CimuMEMS.o CimuMEMS.cpp
g++    -c -o CimuPC.o CimuPC.cpp
g++    -c -o CtmpsenTilt.o CtmpsenTilt.cpp
g++    -c -o CMotorTilt.o CMotorTilt.cpp
/usr/bin/arm-linux-gnueabihf-gcc -o test main.o CAcquireData.o CController.o CDriver.o CFactory.o CSensorFactory.o CDriverFactory.o CSensor.o CimuMEMS.o CimuPC.o CtmpsenTilt.o CMotorTilt.o -lstdc++ -lspidev-lib++ -lpthread -Wall -std=c++11 -I ./include
main.o: file not recognized: file format not recognized
collect2: error: ld returned 1 exit status
make: *** [Makefile:18: test] Error 1

I got the cross compiler from the Ubuntu 20.04 package gcc-arm-linux-gnueabihf.

Why do I get a linker error about it not recognizing the file format for main.o?

Answer 1

There are no build rules for the object files you list as dependencies, so Make compiles them using its default rule, using g++, i.e. the native compiler, not the cross-compiler. You then try to link the object files together using the cross-compiler, which won't work because they were compiled using the native compiler.

---

If you want painless cross-compilation, don't write Makefiles by hand. Use a build system generator like CMake, which supports cross-compilation out of the box.

For example:

CMakeLists.txt

cmake_minimum_required(VERSION 3.16)
project(mistyron VERSION 0.1.0)

add_executable(mistyron
    "src/main.cpp"
    "src/CAcquireData.cpp"
    # ...
)
target_include_directories(mistyron PRIVATE "include")
target_compile_features(mistyron PRIVATE cxx_std_11)
target_compile_options(mistyron PRIVATE -Wall -Wextra)

find_package(Threads REQUIRED)
target_link_libraries(mistyron PRIVATE Threads::Threads spidev-lib++)

install(TARGETS mistyron)

This assumes a directory structure like this:

├── include
│   ├── CAcquireData.hpp
│   └── ...
├── src
│   ├── CAcquireData.cpp
│   ├── ...
│   └── main.cpp
└── CMakeLists.txt

Then create a sysroot for your project, this is a folder with all the system libraries and other dependencies for your target.

cp -a path/to/your/toolchains/sysroot ~/my-sysroot
chmod -R u+w ~/my-sysroot # make it writable

Then tell CMake how to cross-compile for the Raspberry Pi by creating a toolchain file.

arm-linux-gnueabihf.toolchain.cmake

set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR arm)

set(CMAKE_SYSROOT "$ENV{HOME}/my-sysroot")
set(CMAKE_FIND_ROOT_PATH ${CMAKE_SYSROOT})
set(CMAKE_LIBRARY_ARCHITECTURE arm-linux-gnueabihf)

set(cross "arm-linux-gnueabihf")
set(CMAKE_C_COMPILER ${cross}-gcc)
set(CMAKE_CXX_COMPILER ${cross}-g++)

set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE armhf)

Note how the CMAKE_SYSROOT variable is set to the path to the sysroot you created earlier.

Then install your dependencies to the sysroot, in this case, you would do something like this:

pushd ~/Downloads
git clone https://github.com/milekium/spidev-lib.git
pushd spidev-lib
cmake -Bbuild \
    -DCMAKE_BUILD_TYPE=RelWithDebInfo \
    -DCMAKE_TOOLCHAIN_FILE=path/to/arm-linux-gnueabihf.toolchain.cmake \
    -DCMAKE_STAGING_PREFIX=$HOME/my-sysroot/usr
cmake --build build -j
cmake --install build
popd
popd

Finally build your own program:

cmake -S. -Bbuild \
    -DCMAKE_BUILD_TYPE=RelWithDebInfo \
    -DCMAKE_TOOLCHAIN_FILE=path/to/arm-linux-gnueabihf.toolchain.cmake \
    -DCMAKE_STAGING_PREFIX=$HOME/my-staging-dir
cmake --build build  -j
cmake --install build

Your program will be in ~/my-staging-dir/bin/mistyron.

---

An easy way to get reproducible builds is to use a Docker container.

For example, using the toolchain from https://github.com/tttapa/docker-arm-cross-toolchain:

docker/Dockerfile

FROM ghcr.io/tttapa/docker-arm-cross-toolchain:armv6-rpi-linux-gnueabihf-0.0.8

# Install CMake
USER root
RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && \
    apt-get install -y cmake make ninja-build && \
    apt-get clean autoclean && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*
USER develop

# Create a sysroot
ENV SYSROOT_DIR=/home/develop/armv6-rpi-sysroot
RUN cp -a ${TOOLCHAIN_PATH}/armv6-rpi-linux-gnueabihf/sysroot ${SYSROOT_DIR} && \
    chmod -R u+w ${SYSROOT_DIR}

# Copy the CMake Toolchain file
COPY armv6-rpi-linux-gnueabihf.toolchain.cmake .
ENV TOOLCHAIN_FILE=/home/develop/armv6-rpi-linux-gnueabihf.toolchain.cmake

# Install spidev-lib
RUN git clone https://github.com/milekium/spidev-lib.git && \
    cd spidev-lib && \
    cmake -Bbuild \
        -G Ninja \
        -DCMAKE_BUILD_TYPE=RelWithDebInfo \
        -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} \
        -DCMAKE_STAGING_PREFIX=${SYSROOT_DIR}/usr && \
    cmake --build build -j && \
    cmake --install build

docker/armv6-rpi-linux-gnueabihf.toolchain.cmake

set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR arm)

set(CMAKE_SYSROOT $ENV{SYSROOT_DIR})
set(CMAKE_FIND_ROOT_PATH ${CMAKE_SYSROOT})
set(CMAKE_LIBRARY_ARCHITECTURE arm-linux-gnueabihf)

set(cross "armv6-rpi-linux-gnueabihf")
set(CMAKE_C_COMPILER ${cross}-gcc)
set(CMAKE_CXX_COMPILER ${cross}-g++)
set(CMAKE_Fortran_COMPILER ${cross}-gfortran)

set(ARCH_FLAGS "-mcpu=arm1176jzf-s")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ARCH_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARCH_FLAGS}")
set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${ARCH_FLAGS}")

set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE armhf)

build.sh

#!/usr/bin/env bash
cd "$(dirname "${BASH_SOURCE[0]}")"
set -ex

# Build the Docker container for building
docker build docker -t mistyron/build

# Run the Docker container for building, mounting the directory
# with your project's source code, and then build your project
docker run -i --rm --volume "$PWD:/mnt" mistyron/build << 'EOF'
set -ex
cd /tmp
cmake -S/mnt -Bbuild \
    -G Ninja \
    -DCMAKE_BUILD_TYPE=RelWithDebInfo \
    -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} \
    -DCMAKE_STAGING_PREFIX=/mnt/staging
cmake --build build  -j
cmake --install build
EOF

Assuming this directory structure:

├── docker
│   ├── armv6-rpi-linux-gnueabihf.toolchain.cmake
│   └── Dockerfile
├── include
│   ├── CAcquireData.hpp
│   └── ...
├── src
│   ├── CAcquireData.cpp
│   ├── ...
│   └── main.cpp
├── build.sh
└── CMakeLists.txt

Then simply run

./build.sh # make sure it's executable, use chmod +x if necessary

and you'll be able to find your program in staging/bin/mistyron.

To execute it on the Pi, you may have to install some additional libraries (e.g. the correct version of the C++ library, your own third-party dependencies, etc.). More info here https://tttapa.github.io/Pages/Raspberry-Pi/C++-Development-RPiOS/Development-setup.html#install-the-c-standard-library-to-the-pi.

---

Edit to address comment about 64-bit support:

1. Use a 64-bit toolchain, for example, use ghcr.io/tttapa/docker-arm-cross-toolchain:aarch64-rpi3-linux-gnu-0.0.8
2. Tweak some of the paths that still point to the armv6 toolchain, sysroot, etc.
3. Use an appropriate toolchain file:

set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR aarch64)

set(CMAKE_SYSROOT $ENV{SYSROOT_DIR})
set(CMAKE_FIND_ROOT_PATH ${CMAKE_SYSROOT}) 
set(CMAKE_LIBRARY_ARCHITECTURE aarch64-linux-gnu)
set(CMAKE_STAGING_PREFIX $ENV{HOME}/RPi-dev/staging-aarch64-rpi3)

set(cross "aarch64-rpi3-linux-gnu")
set(CMAKE_C_COMPILER ${cross}-gcc)
set(CMAKE_CXX_COMPILER ${cross}-g++)
set(CMAKE_Fortran_COMPILER ${cross}-gfortran)

set(ARCH_FLAGS "-mcpu=cortex-a53+crc+simd")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ARCH_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARCH_FLAGS}")
set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${ARCH_FLAGS}")

set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE arm64)

The CPack architecture option has no effect on the compilation itself, it's just used for packaging (which I didn't use in this example).

Answer 2

The problem is that you set CC for the cross compiler, but you need to set CXX to set the C++ cross compiler. Since you didn't change that, it used the native C++ compiler, and the object files are in the wrong format.