Update to bootloader-locator 0.0.3; cargo fmt; add QEMU run args

This allows us to trigger a branch build from the master branch on schedule.

.cargo/config

@@ -1,5 +0,0 @@
-[build]
-target = "x86_64-blog_os.json"
-
-[target.'cfg(target_os = "none")']
-runner = "bootimage runner"

.cargo/config.toml

@@ -0,0 +1,11 @@
+[unstable]
+# build-std = ["core", "compiler_builtins"]
+
+[build]
+# target = "x86_64-blog_os.json"
+
+[target.'cfg(target_os = "none")']
+runner = "cargo run --manifest-path disk_image/Cargo.toml --bin runner --quiet"
+
+[alias]
+bootimage = "run --manifest-path disk_image/Cargo.toml --bin disk_image --quiet"

.github/workflows/build-code.yml

@@ -10,11 +10,11 @@ on:
   schedule:
     - cron: '40 3 * * *'   # every day at 3:40
   pull_request:
+  workflow_dispatch:
 
 jobs:
-  test:
+  check:
-    name: "Test"
+    name: Check
-
     strategy:
       matrix:
         platform: [
@@ -22,75 +22,111 @@ jobs:
           macos-latest,
           windows-latest
         ]
-
     runs-on: ${{ matrix.platform }}
-    timeout-minutes: 15
-
     steps:
-    - name: "Checkout Repository"
+      - name: Checkout Repository
-      uses: actions/checkout@v1
+        uses: actions/checkout@v2
+      - name: Install Rust Toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+      - name: Install `rust-src` Rustup Component
+        run: rustup component add rust-src
+      - name: Run `cargo check`
+        uses: actions-rs/cargo@v1
+        with:
+          command: check
 
-    - name: Install Rustup
+  test:
-      run: |
+    name: Test
-        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly
+    strategy:
-        echo ::add-path::$HOME/.cargo/bin
+      matrix:
-      if: runner.os == 'macOS'
+        platform: [
+          ubuntu-latest,
+          macos-latest,
+          windows-latest
+        ]
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - name: Install Rust Toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+      - name: Install bootimage
+        run: cargo install bootimage --debug
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+      - name: Install Rustup Components
+        run: rustup component add rust-src llvm-tools-preview
+      - name: Run `cargo bootimage`
+        uses: actions-rs/cargo@v1
+        with:
+          command: bootimage
 
-    - name: "Print Rust Version"
+      # install QEMU
-      run: |
+      - name: Install QEMU (Linux)
-        rustc -Vv
+        run: sudo apt update && sudo apt install qemu-system-x86
-        cargo -Vv
+        if: runner.os == 'Linux'
+      - name: Install QEMU (macOS)
+        run: brew install qemu
+        if: runner.os == 'macOS'
+        env:
+          HOMEBREW_NO_AUTO_UPDATE: 1
+          HOMEBREW_NO_BOTTLE_SOURCE_FALLBACK: 1
+          HOMEBREW_NO_INSTALL_CLEANUP: 1
+      - name: Install Scoop (Windows)
+        run: |
+          Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')
+          echo ::add-path::$HOME\scoop\shims
+        if: runner.os == 'Windows'
+        shell: pwsh
+      - name: Install QEMU (Windows)
+        run: scoop install qemu
+        if: runner.os == 'Windows'
+        shell: pwsh
+      - name: "Print QEMU Version"
+        run: qemu-system-x86_64 --version
 
-    - name: "Install Rustup Components"
+      - name: Run `cargo test`
-      run: rustup component add rust-src llvm-tools-preview
+        uses: actions-rs/cargo@v1
-    - name: "Install cargo-xbuild"
+        with:
-      run: cargo install cargo-xbuild --debug
+          command: test
-    - name: "Install bootimage"
-      run: cargo install bootimage --debug
-
-    - name: "Run cargo xbuild"
-      run: cargo xbuild
-    - name: "Create Bootimage"
-      run: cargo bootimage
-
-     # install QEMU
-    - name: Install QEMU (Linux)
-      run: sudo apt update && sudo apt install qemu-system-x86
-      if: runner.os == 'Linux'
-    - name: Install QEMU (macOS)
-      run: brew install qemu
-      if: runner.os == 'macOS'
-      env:
-        HOMEBREW_NO_AUTO_UPDATE: 1
-        HOMEBREW_NO_BOTTLE_SOURCE_FALLBACK: 1
-        HOMEBREW_NO_INSTALL_CLEANUP: 1
-    - name: Install Scoop (Windows)
-      run: |
-        Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')
-        echo ::add-path::$HOME\scoop\shims
-      if: runner.os == 'Windows'
-      shell: pwsh
-    - name: Install QEMU (Windows)
-      run: scoop install qemu
-      if: runner.os == 'Windows'
-      shell: pwsh
-    - name: "Print QEMU Version"
-      run: qemu-system-x86_64 --version
-
-    - name: "Run cargo xtest"
-      run: cargo xtest
 
   check_formatting:
-    name: "Check Formatting"
+    name: Check Formatting
     runs-on: ubuntu-latest
-    timeout-minutes: 2
     steps:
-    - uses: actions/checkout@v1
+      - name: Checkout Repository
-    - name: "Use the latest Rust nightly with rustfmt"
+        uses: actions/checkout@v2
-      uses: actions-rs/toolchain@v1
+      - name: Install Rust Toolchain
-      with:
+        uses: actions-rs/toolchain@v1
-          toolchain: nightly
+        with:
           profile: minimal
+          toolchain: nightly
           components: rustfmt
           override: true
-    - run: cargo fmt -- --check
+      - name: Run `cargo fmt`
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  clippy:
+    name: Clippy
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+      - name: Install Rust Toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          components: clippy, rust-src
+          override: true
+      - name: Run `cargo clippy`
+        uses: actions-rs/cargo@v1
+        with:
+          command: clippy

Cargo.lock

@@ -1,5 +1,11 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+[[package]]
+name = "anyhow"
+version = "1.0.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b602bfe940d21c130f3895acd65221e8a61270debe89d628b9cb4e3ccb8569b"
+
 [[package]]
 name = "bit_field"
 version = "0.9.0"
@@ -18,6 +24,7 @@ version = "0.1.0"
 dependencies = [
  "bootloader",
  "lazy_static",
+ "rlibc",
  "spin",
  "uart_16550",
  "volatile",
@@ -26,9 +33,49 @@ dependencies = [
 
 [[package]]
 name = "bootloader"
-version = "0.9.3"
+version = "0.9.8"
+
+[[package]]
+name = "bootloader-locator"
+version = "0.0.3"
+dependencies = [
+ "cargo_metadata",
+ "locate-cargo-manifest",
+ "thiserror",
+]
+
+[[package]]
+name = "cargo_metadata"
+version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44ac0bdf4930c3c4d7f0d04eb6f15d7dcb9d5972b1ff9cd2bee0128112260fc7"
+checksum = "89fec17b16f1ac67908af82e47d0a90a7afd0e1827b181cd77504323d3263d35"
+dependencies = [
+ "semver",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "disk_image"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "bootloader-locator",
+ "locate-cargo-manifest",
+ "runner-utils",
+]
+
+[[package]]
+name = "itoa"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6"
+
+[[package]]
+name = "json"
+version = "0.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd"
 
 [[package]]
 name = "lazy_static"
@@ -39,12 +86,144 @@ dependencies = [
  "spin",
 ]
 
+[[package]]
+name = "libc"
+version = "0.2.76"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "755456fae044e6fa1ebbbd1b3e902ae19e73097ed4ed87bb79934a867c007bc3"
+
+[[package]]
+name = "locate-cargo-manifest"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d264d672563b20c0a2eb9045f4072e1352002f6c316cc0677bed8b767e17b7e1"
+dependencies = [
+ "json",
+ "thiserror",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12"
+dependencies = [
+ "unicode-xid",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rlibc"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc874b127765f014d792f16763a81245ab80500e2ad921ed4ee9e82481ee08fe"
+
+[[package]]
+name = "runner-utils"
+version = "0.0.2"
+dependencies = [
+ "thiserror",
+ "wait-timeout",
+]
+
+[[package]]
+name = "ryu"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
+
+[[package]]
+name = "semver"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "394cec28fa623e00903caf7ba4fa6fb9a0e260280bb8cdbbba029611108a0190"
+dependencies = [
+ "semver-parser",
+ "serde",
+]
+
+[[package]]
+name = "semver-parser"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
+
+[[package]]
+name = "serde"
+version = "1.0.115"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e54c9a88f2da7238af84b5101443f0c0d0a3bbdc455e34a5c9497b1903ed55d5"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.115"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "609feed1d0a73cc36a0182a840a9b37b4a82f0b1150369f0536a9e3f2a31dc48"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.57"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "164eacbdb13512ec2745fb09d51fd5b22b0d65ed294a1dcf7285a360c80a675c"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
 [[package]]
 name = "spin"
 version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
 
+[[package]]
+name = "syn"
+version = "1.0.39"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "891d8d6567fe7c7f8835a3a98af4208f3846fba258c1bc3c31d6e506239f11f9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-xid",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7dfdd070ccd8ccb78f4ad66bf1982dc37f620ef696c6b5028fe2ed83dd3d0d08"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "uart_16550"
 version = "0.2.7"
@@ -56,10 +235,25 @@ dependencies = [
 ]
 
 [[package]]
-name = "volatile"
+name = "unicode-xid"
-version = "0.2.6"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6af0edf5b4faacc31fc51159244d78d65ec580f021afcef7bd53c04aeabc7f29"
+checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
+
+[[package]]
+name = "volatile"
+version = "0.4.0-alpha.00"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a1da81878b88e2b57ca8b3d62c0c3e9979d9531cfe0e6d1c094174d8458eac6"
+
+[[package]]
+name = "wait-timeout"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
+dependencies = [
+ "libc",
+]
 
 [[package]]
 name = "x86_64"

Cargo.toml

@@ -4,13 +4,19 @@ version = "0.1.0"
 authors = ["Philipp Oppermann <dev@phil-opp.com>"]
 edition = "2018"
 
+[workspace]
+members = [
+    "disk_image",
+]
+
 [[test]]
 name = "should_panic"
 harness = false
 
 [dependencies]
-bootloader = "0.9.3"
+bootloader = { path = "../../uefi-test"}
-volatile = "0.2.6"
+rlibc = "1.0.0"
+volatile = "0.4.0-alpha"
 spin = "0.5.2"
 x86_64 = "0.11.0"
 uart_16550 = "0.2.0"

README.md

@@ -10,19 +10,23 @@ This repository contains the source code for the [Testing][post] post of the [Wr
 
 ## Building
 
-You need a nightly Rust compiler. First you need to install the `cargo-xbuild` and `bootimage` tools:
+This project requires a nightly version of Rust because it uses some unstable features. At least nightly _2020-07-15_ is required for building. You might need to run `rustup update nightly --force` to update to the latest nightly even if some components such as `rustfmt` are missing it.
+
+You can build the project by running:
 
 ```
-cargo install cargo-xbuild bootimage
+cargo build
 ```
 
-Then you can build the project by running:
+To create a bootable disk image from the compiled kernel, you need to install the [`bootimage`] tool:
+
+[`bootimage`]: https://github.com/rust-osdev/bootimage
 
 ```
-cargo xbuild
+cargo install bootimage
 ```
 
-To create a bootable disk image, run:
+After installing, you can create the bootable disk image by running:
 
 ```
 cargo bootimage
@@ -39,10 +43,10 @@ You can run the disk image in [QEMU] through:
 [QEMU]: https://www.qemu.org/
 
 ```
-cargo xrun
+cargo run
 ```
 
-Of course [QEMU] needs to be installed for this.
+[QEMU] and the [`bootimage`] tool need to be installed for this.
 
 You can also write the image to an USB stick for booting it on a real machine. On Linux, the command for this is:
 

disk_image/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "disk_image"
+version = "0.1.0"
+authors = ["Philipp Oppermann <dev@phil-opp.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.32"
+bootloader-locator = {path = "../../../bootloader-locator"}
+runner-utils = { path = "../../../runner-utils/" }
+locate-cargo-manifest = "0.2.0"

disk_image/src/bin/runner.rs

@@ -0,0 +1,57 @@
+use anyhow::anyhow;
+use std::{
+    path::PathBuf,
+    process::{Command, ExitStatus},
+    time::Duration,
+};
+
+const RUN_ARGS: &[&str] = &["-d", "int", "--no-reboot", "-s"];
+const TEST_ARGS: &[&str] = &[
+    "-device",
+    "isa-debug-exit,iobase=0xf4,iosize=0x04",
+    "-serial",
+    "stdio",
+    "-display",
+    "none",
+    "--no-reboot",
+];
+const TEST_TIMEOUT_SECS: u64 = 10;
+
+fn main() -> anyhow::Result<()> {
+    let kernel_binary_path = {
+        let path = PathBuf::from(std::env::args().nth(1).unwrap());
+        path.canonicalize()?
+    };
+
+    let disk_image = disk_image::create_disk_image(&kernel_binary_path, true)?;
+
+    let mut run_cmd = Command::new("qemu-system-x86_64");
+    run_cmd
+        .arg("-drive")
+        .arg(format!("format=raw,file={}", disk_image.display()));
+
+    let binary_kind = runner_utils::binary_kind(&kernel_binary_path);
+    if binary_kind.is_test() {
+        run_cmd.args(TEST_ARGS);
+
+        let exit_status = run_test_command(run_cmd)?;
+        match exit_status.code() {
+            Some(33) => {} // success
+            other => return Err(anyhow!("Test failed (exit code: {:?})", other)),
+        }
+    } else {
+        run_cmd.args(RUN_ARGS);
+
+        let exit_status = run_cmd.status()?;
+        if !exit_status.success() {
+            std::process::exit(exit_status.code().unwrap_or(1));
+        }
+    }
+
+    Ok(())
+}
+
+fn run_test_command(mut cmd: Command) -> anyhow::Result<ExitStatus> {
+    let status = runner_utils::run_with_timeout(&mut cmd, Duration::from_secs(TEST_TIMEOUT_SECS))?;
+    Ok(status)
+}

disk_image/src/lib.rs

@@ -0,0 +1,45 @@
+use anyhow::anyhow;
+use std::{
+    path::{Path, PathBuf},
+    process::Command,
+};
+
+pub fn create_disk_image(kernel_binary_path: &Path, bios_only: bool) -> anyhow::Result<PathBuf> {
+    let bootloader_info = bootloader_locator::locate_bootloader()?;
+    let kernel_manifest_path = &bootloader_info.kernel_manifest_path;
+
+    let mut build_cmd = Command::new(env!("CARGO"));
+    build_cmd.current_dir(bootloader_info.package.manifest_path.parent().unwrap());
+    build_cmd.arg("builder");
+    build_cmd
+        .arg("--kernel-manifest")
+        .arg(&kernel_manifest_path);
+    build_cmd.arg("--kernel-binary").arg(&kernel_binary_path);
+    build_cmd
+        .arg("--target-dir")
+        .arg(kernel_manifest_path.parent().unwrap().join("target"));
+    build_cmd
+        .arg("--out-dir")
+        .arg(kernel_binary_path.parent().unwrap());
+    build_cmd.arg("--quiet");
+    if bios_only {
+        build_cmd.arg("--firmware").arg("bios");
+    }
+
+    if !build_cmd.status()?.success() {
+        return Err(anyhow!("build failed"));
+    }
+
+    let kernel_binary_name = kernel_binary_path.file_name().unwrap().to_str().unwrap();
+    let disk_image = kernel_binary_path
+        .parent()
+        .unwrap()
+        .join(format!("bootimage-bios-{}.bin", kernel_binary_name));
+    if !disk_image.exists() {
+        return Err(anyhow!(
+            "Disk image does not exist at {} after bootloader build",
+            disk_image.display()
+        ));
+    }
+    Ok(disk_image)
+}

disk_image/src/main.rs

@@ -0,0 +1,40 @@
+use anyhow::anyhow;
+use std::process::Command;
+
+const TARGET_NAME: &str = "x86_64-blog_os";
+const KERNEL_BINARIES: &[&str] = &["blog_os"];
+
+fn main() -> anyhow::Result<()> {
+    // build all binaries
+    let mut build_cmd = Command::new(env!("CARGO"));
+    build_cmd.arg("build");
+    build_cmd.arg("--release");
+    build_cmd.arg("-Zbuild-std=core");
+    build_cmd
+        .arg("--target")
+        .arg(format!("{}.json", TARGET_NAME));
+    if !build_cmd.status()?.success() {
+        return Err(anyhow!("build failed"));
+    };
+
+    let kernel_manifest = locate_cargo_manifest::locate_manifest()?;
+    let target_dir_root = kernel_manifest.parent().unwrap().join("target");
+    let target_dir = target_dir_root.join(TARGET_NAME).join("release");
+
+    for binary_name in KERNEL_BINARIES {
+        let binary_path = {
+            let path = target_dir.join(binary_name);
+            path.canonicalize()?
+        };
+
+        let disk_image = disk_image::create_disk_image(&binary_path, false)?;
+
+        println!(
+            "Created disk image for binary {} at {}",
+            binary_name,
+            disk_image.display()
+        );
+    }
+
+    Ok(())
+}

src/lib.rs

@@ -4,10 +4,12 @@
 #![test_runner(crate::test_runner)]
 #![reexport_test_harness_main = "test_main"]
 
+extern crate rlibc;
+
 use core::panic::PanicInfo;
 
 pub mod serial;
-pub mod vga_buffer;
+//pub mod vga_buffer;
 
 pub trait Testable {
     fn run(&self) -> ();

src/main.rs

@@ -4,24 +4,36 @@
 #![test_runner(blog_os::test_runner)]
 #![reexport_test_harness_main = "test_main"]
 
-use blog_os::println;
+use blog_os::serial_println;
-use core::panic::PanicInfo;
+use core::{panic::PanicInfo, ptr, slice};
 
 #[no_mangle]
-pub extern "C" fn _start() -> ! {
+pub extern "C" fn _start(boot_info: &'static mut bootloader::boot_info::BootInfo) -> ! {
-    println!("Hello World{}", "!");
-
     #[cfg(test)]
     test_main();
 
-    loop {}
+    let mut framebuffer = {
+        let ptr = boot_info.framebuffer.start_addr as *mut u8;
+        let slice = unsafe { slice::from_raw_parts_mut(ptr, boot_info.framebuffer.len) };
+        volatile::Volatile::new(slice)
+    };
+
+    //serial_println!("Hello World{}", "!");
+
+    for i in 0..boot_info.framebuffer.len {
+        framebuffer.index_mut(i).write(0x99);
+    }
+
+    loop {
+        x86_64::instructions::hlt();
+    }
 }
 
 /// This function is called on panic.
 #[cfg(not(test))]
 #[panic_handler]
 fn panic(info: &PanicInfo) -> ! {
-    println!("{}", info);
+    serial_println!("{}", info);
     loop {}
 }
 

tests/basic_boot.rs

@@ -4,7 +4,6 @@
 #![test_runner(blog_os::test_runner)]
 #![reexport_test_harness_main = "test_main"]
 
-use blog_os::println;
 use core::panic::PanicInfo;
 
 #[no_mangle] // don't mangle the name of this function
@@ -21,5 +20,5 @@ fn panic(info: &PanicInfo) -> ! {
 
 #[test_case]
 fn test_println() {
-    println!("test_println output");
+    //println!("test_println output");
 }