Skip to content

Commit

Permalink
Run uas blacklist earlier in boot process before anything happens wit…
Browse files Browse the repository at this point in the history
…h the SSD
  • Loading branch information
lukechilds committed Mar 22, 2024
1 parent 79d37f1 commit 79fe570
Show file tree
Hide file tree
Showing 4 changed files with 112 additions and 89 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,15 @@ main () {
exit 1
fi

# Check if device is running with uas driver, if so balcklist it and reboot
echo "Checking if we need to blacklist UAS"
blacklist_uas_output=$(umbreld blacklist-uas || true)
# Check output includes text "mount-script-halt" which means we are rebooting and should not mount
if [[ "${blacklist_uas_output}" == *"mount-script-halt"* ]]; then
echo "UAS was blacklisted and device is rebooting, exiting"
return
fi

# At this point we know there is only one block device attached
block_device=$(list_block_devices)
block_device_path="/dev/${block_device}"
Expand Down
7 changes: 7 additions & 0 deletions packages/umbreld/source/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import camelcaseKeys from 'camelcase-keys'

import {cliClient} from './modules/cli-client.js'
import provision from './modules/provision/provision.js'
import blacklistUas from './modules/blacklist-uas/blacklist-uas.js'

import Umbreld, {type UmbreldOptions} from './index.js'

Expand All @@ -15,6 +16,12 @@ if (process.argv.includes('provision-os')) {
process.exit(0)
}

// Blacklists uas drivers early in the boot process
if (process.argv.includes('blacklist-uas')) {
await blacklistUas()
process.exit(0)
}

// Quick trpc client for testing
if (process.argv.includes('client')) {
const clientIndex = process.argv.indexOf('client')
Expand Down
89 changes: 0 additions & 89 deletions packages/umbreld/source/index.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
import path from 'node:path'
import {setTimeout} from 'node:timers/promises'

import {globby} from 'globby'
import fse from 'fs-extra'
import {$} from 'execa'

// @ts-expect-error I can't get tsconfig setup in a way that allows this without breaking other things.
Expand Down Expand Up @@ -89,89 +87,6 @@ export default class Umbreld {
}
}

// By default Linux uses the UAS driver for most devices. This causes major
// stability problems on the Raspberry Pi 4, not due to issues with UAS, but due
// to devices running in UAS mode using much more power. The Pi can't reliably
// provide enough power to the USB port and the entire system experiences
// extreme instability. By blacklisting all devices from the UAS driver on first
// and then rebooting we fall back to the mass-storage driver, which results in
// decreased performance, but lower power usage, and much better system stability.
// TODO: Move this to a system module
async blacklistUASDriver() {
try {
const justDidRebootFile = '/umbrel-just-did-reboot'
// Only run on Raspberry Pi 4
const {deviceId} = await detectDevice()
if (deviceId !== 'pi-4') return
this.logger.log('Checking for UAS devices to blacklist')
const blacklist = []
// Get all USB device uevent files
const usbDeviceUeventFiles = await globby('/sys/bus/usb/devices/*/uevent')
for (const ueventFile of usbDeviceUeventFiles) {
const uevent = await fse.readFile(ueventFile, 'utf8')
if (!uevent.includes('DRIVER=uas')) continue
const [vendorId, productId] = uevent
.split('\n')
.find((line) => line?.startsWith('PRODUCT='))
.replace('PRODUCT=', '')
.split('/')
const deviceId = `${vendorId}:${productId}`
this.logger.log(`UAS device found ${deviceId}`)
blacklist.push(deviceId)
}

// Don't reboot if we don't have any UAS devices
if (blacklist.length === 0) {
this.logger.log('No UAS devices found!')
await fse.remove(justDidRebootFile)
return
}

// Check we're not in a boot loop
if (await fse.pathExists(justDidRebootFile)) {
this.logger.log('We just rebooted, we could be in a bootloop, skipping reboot')
return
}

// Read current cmdline
this.logger.log(`Applying quirks to cmdline.txt`)
let cmdline = await fse.readFile('/boot/cmdline.txt', 'utf8')

// Don't apply quirks if they're already applied
const quirksAlreadyApplied = blacklist.every((deviceId) => cmdline.includes(`${deviceId}:u`))
if (quirksAlreadyApplied) {
this.logger.log('UAS quirks already applied, skipping')
return
}

// Remove any current quirks
cmdline = cmdline
.trim()
.split(' ')
.filter((flag) => !flag.startsWith('usb-storage.quirks='))
.join(' ')
// Add new quirks
const quirks = blacklist.map((deviceId) => `${deviceId}:u`).join(',')
cmdline = `${cmdline} usb-storage.quirks=${quirks}`

// Remount /boot as writable
await $`mount -o remount,rw /boot`
// Write new cmdline
await fse.writeFile('/boot/cmdline.txt', cmdline)

// Reboot the system
this.logger.log(`Rebooting`)
// We need to make sure we commit before rebooting otherwise
// OTA updates will get instantly rolled back.
await commitOsPartition(this)
await fse.writeFile(justDidRebootFile, cmdline)
await reboot()
return true
} catch (error) {
this.logger.error(`Failed to blacklist UAS driver: ${(error as Error).message}`)
}
}

// Wait for system time to be synced for up to the number of seconds passed in.
// We need this on Raspberry Pi since it doesn' have a persistent real time clock.
// It avoids race conditions where umbrelOS starts making network requests before
Expand Down Expand Up @@ -212,10 +127,6 @@ export default class Umbreld {
// If we've successfully booted then commit to the current OS partition
commitOsPartition(this)

// Blacklist UAS driver for Raspberry Pi 4
const isRebooting = await this.blacklistUASDriver()
if (isRebooting === true) return // Don't let the server start if we're rebooting

// Set ondemand cpu governer for Raspberry Pi
this.setupPiCpuGoverner()

Expand Down
96 changes: 96 additions & 0 deletions packages/umbreld/source/modules/blacklist-uas/blacklist-uas.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
import {globby} from 'globby'
import fse from 'fs-extra'
import {$} from 'execa'

// By default Linux uses the UAS driver for most devices. This causes major
// stability problems on the Raspberry Pi 4, not due to issues with UAS, but due
// to devices running in UAS mode using much more power. The Pi can't reliably
// provide enough power to the USB port and the entire system experiences
// extreme instability. By blacklisting all devices from the UAS driver on first
// and then rebooting we fall back to the mass-storage driver, which results in
// decreased performance, but lower power usage, and much better system stability.
//
// We use console.err for logs so they appear in umbrel-external-storage logs and
// console.log to signal to the mount script that we're rebooting.
export default async function blacklistUASDriver() {
try {
console.error('Checking for UAS devices to blacklist')
const justDidRebootFile = '/umbrel-just-did-reboot'
// Only run on Raspberry Pi 4
const cpuInfo = await fse.readFile('/proc/cpuinfo')
if (!cpuInfo.includes('Raspberry Pi 4 ')) {
console.error('Not running on Pi 4, exiting...')
return
}
console.error('Running on Pi 4')
const blacklist = []
// Get all USB device uevent files
const usbDeviceUeventFiles = await globby('/sys/bus/usb/devices/*/uevent')
for (const ueventFile of usbDeviceUeventFiles) {
const uevent = await fse.readFile(ueventFile, 'utf8')
if (!uevent.includes('DRIVER=uas')) continue
const [vendorId, productId] = uevent
.split('\n')
.find((line) => line?.startsWith('PRODUCT='))
.replace('PRODUCT=', '')
.split('/')
const deviceId = `${vendorId}:${productId}`
console.error(`UAS device found ${deviceId}`)
blacklist.push(deviceId)
}

// Don't reboot if we don't have any UAS devices
if (blacklist.length === 0) {
console.error('No UAS devices found!')
await fse.remove(justDidRebootFile)
return
}

// Check we're not in a boot loop
if (await fse.pathExists(justDidRebootFile)) {
console.error('We just rebooted, we could be in a bootloop, skipping reboot')
return
}

// Read current cmdline
console.error(`Applying quirks to cmdline.txt`)
let cmdline = await fse.readFile('/boot/cmdline.txt', 'utf8')

// Don't apply quirks if they're already applied
const quirksAlreadyApplied = blacklist.every((deviceId) => cmdline.includes(`${deviceId}:u`))
if (quirksAlreadyApplied) {
console.error('UAS quirks already applied, skipping')
return
}

// Remove any current quirks
cmdline = cmdline
.trim()
.split(' ')
.filter((flag) => !flag.startsWith('usb-storage.quirks='))
.join(' ')
// Add new quirks
const quirks = blacklist.map((deviceId) => `${deviceId}:u`).join(',')
cmdline = `${cmdline} usb-storage.quirks=${quirks}`

// Remount /boot as writable
await $`mount -o remount,rw /boot`
// Write new cmdline
await fse.writeFile('/boot/cmdline.txt', cmdline)

// Reboot the system
console.error(`Rebooting`)
// We must make exactly this console log so we can detect a reboot in the mount script and halt
console.log('mount-script-halt')
// We need to make sure we commit before rebooting otherwise
// OTA updates will get instantly rolled back.
try {
await $`mender commit`
} catch {}
await fse.writeFile(justDidRebootFile, cmdline)
await $`reboot`
return true
} catch (error) {
console.error(`Failed to blacklist UAS driver: ${(error as Error).message}`)
}
}

0 comments on commit 79fe570

Please sign in to comment.