Merge branch 'develop' into wagonpathfix

.github/workflows/build-windows.yml

@@ -32,6 +32,9 @@ on:
       platform-files:
         type: boolean
         default: true
+      plugins:
+        type: boolean
+        default: true
       common-files:
         type: boolean
         default: true
@@ -104,7 +107,7 @@ jobs:
             win-msvc
       - name: Cross-compile
         env:
-          CMAKE_EXTRA_ARGS: -DBUILD_LIBRARY=${{ inputs.platform-files }} -DBUILD_STONESENSE:BOOL=${{ inputs.stonesense }} -DBUILD_DOCS:BOOL=${{ inputs.docs }} -DBUILD_DOCS_NO_HTML:BOOL=${{ !inputs.html }} -DINSTALL_DATA_FILES:BOOL=${{ inputs.common-files }} -DINSTALL_SCRIPTS:BOOL=${{ inputs.common-files }} -DBUILD_DFLAUNCH:BOOL=${{ inputs.launchdf }} -DBUILD_TESTS:BOOL=${{ inputs.tests }} -DBUILD_XMLDUMP:BOOL=${{ inputs.xml-dump-type-sizes }} ${{ inputs.xml-dump-type-sizes && '-DINSTALL_XMLDUMP:BOOL=1' || '' }}
+          CMAKE_EXTRA_ARGS: -DBUILD_LIBRARY=${{ inputs.platform-files }} -DBUILD_PLUGINS:BOOL=${{ inputs.platform-files && inputs.plugins }} -DBUILD_STONESENSE:BOOL=${{ inputs.stonesense }} -DBUILD_DOCS:BOOL=${{ inputs.docs }} -DBUILD_DOCS_NO_HTML:BOOL=${{ !inputs.html }} -DINSTALL_DATA_FILES:BOOL=${{ inputs.common-files }} -DINSTALL_SCRIPTS:BOOL=${{ inputs.common-files }} -DBUILD_DFLAUNCH:BOOL=${{ inputs.launchdf }} -DBUILD_TESTS:BOOL=${{ inputs.tests }} -DBUILD_XMLDUMP:BOOL=${{ inputs.xml-dump-type-sizes }} ${{ inputs.xml-dump-type-sizes && '-DINSTALL_XMLDUMP:BOOL=1' || '' }}
         run: |
           cd build
           bash -x build-win64-from-linux.sh

.github/workflows/generate-symbols.yml

@@ -8,7 +8,7 @@ on:
         required: true
         default: master
       version:
-        description: DF version
+        description: DF version (can be "auto" if channel is steam)
         required: true
       platform:
         description: Target OS platform
@@ -44,7 +44,7 @@ on:
         type: string
 
 jobs:
-  package:
+  package-linux:
     uses: ./.github/workflows/build-linux.yml
     if: inputs.platform == 'all' || inputs.platform == 'linux'
     with:
@@ -57,12 +57,25 @@ jobs:
       plugins: false
     secrets: inherit
 
+  package-win64:
+    uses: ./.github/workflows/build-windows.yml
+    if: (inputs.platform == 'all' || inputs.platform == 'windows') && inputs.version == 'auto'
+    with:
+      dfhack_ref: ${{ github.ref }}
+      structures_ref: ${{ inputs.structures_ref }}
+      artifact-name: dfhack-symbols-windows64-build
+      append-date-and-hash: false
+      cache-id: release
+      cache-readonly: true
+      plugins: false
+    secrets: inherit
+
   generate-linux:
     name: Generate linux64 symbols
     runs-on: ubuntu-latest
     if: inputs.platform == 'all' || inputs.platform == 'linux'
     needs:
-    - package
+    - package-linux
     steps:
     - name: Install dependencies
       run: |
@@ -119,11 +132,28 @@ jobs:
           "+app_update 975370 $BETA_PARAMS validate" \
           +quit
         tar xjf dfhack-symbols-linux64-build.tar.bz2 -C DF_steam
-        xml/symbols_gen_linux.sh ${{ inputs.version }} STEAM DF_steam
+        xml/symbols_gen_linux.sh ${{ inputs.version == 'auto' && '50.0' || inputs.version }} STEAM DF_steam
+        if [ "${{ inputs.version }}" = "auto" ]; then
+          while pgrep dwarfort; do
+            echo "waiting for DF to exit"
+            sleep 0.5
+          done
+          cp xml/symbols.xml DF_steam/hack
+          cd DF_steam
+          DFHACK_DISABLE_CONSOLE=1 ./dfhack &
+          while ! ./dfhack-run lua 'print(scr)' | fgrep 'viewscreen_titlest' 2>/dev/null; do
+            echo "waiting for DF to start"
+            sleep 0.5
+          done
+          df_ver=`./dfhack-run lua 'print(dfhack.gui.getDFViewscreen(true).str_version)' | ansifilter`
+          echo "Found version string: '$df_ver'"
+          sed -i "s/v0.50.0 linux64 STEAM/v0.$df_ver linux64 STEAM/" ../xml/symbols.xml
+          ./dfhack-run die || true
+        fi
 
     # Itch
     - name: Generate Itch symbols
-      if: inputs.channel == 'all' || inputs.channel == 'itch'
+      if: (inputs.channel == 'all' || inputs.channel == 'itch') && inputs.version != 'auto'
       env:
         DISPLAY: :0
         ITCH_API_KEY: ${{ secrets.ITCH_API_KEY }}
@@ -139,7 +169,7 @@ jobs:
 
     # Classic
     - name: Generate Classic symbols
-      if: inputs.channel == 'all' || inputs.channel == 'classic'
+      if: (inputs.channel == 'all' || inputs.channel == 'classic') && inputs.version != 'auto'
       env:
         DISPLAY: :0
       run: |
@@ -227,11 +257,11 @@ jobs:
           +login $STEAM_USERNAME \
           "+app_update 975370 $BETA_PARAMS validate" \
           +quit
-        xml/symbols_gen_windows.sh ${{ inputs.version }} STEAM DF_steam
+        xml/symbols_gen_windows.sh ${{ inputs.version == 'auto' && '50.0' || inputs.version }} STEAM DF_steam
 
     # Itch
     - name: Generate Itch symbols
-      if: inputs.channel == 'all' || inputs.channel == 'itch'
+      if: (inputs.channel == 'all' || inputs.channel == 'itch') && inputs.version != 'auto'
       env:
         ITCH_API_KEY: ${{ secrets.ITCH_API_KEY }}
       run: |
@@ -245,7 +275,7 @@ jobs:
 
     # Classic
     - name: Generate Classic symbols
-      if: inputs.channel == 'all' || inputs.channel == 'classic'
+      if: (inputs.channel == 'all' || inputs.channel == 'classic') && inputs.version != 'auto'
       run: |
         mkdir DF_classic
         minor=$(echo "${{ inputs.version }}" | cut -d. -f1)
@@ -272,24 +302,106 @@ jobs:
         commit_user_name: DFHack-Urist via GitHub Actions
         commit_user_email: [email protected]
 
+  auto-ver-windows:
+    name: Autodetect DF version string (Windows)
+    if: (inputs.platform == 'all' || inputs.platform == 'windows') && (inputs.channel == 'all' || inputs.channel == 'steam') && inputs.version == 'auto'
+    needs:
+      - package-win64
+      - generate-windows
+    runs-on: windows-latest
+    steps:
+    - name: Clone structures
+      uses: actions/checkout@v4
+      with:
+        repository: DFHack/df-structures
+        ref: ${{ inputs.structures_ref }}
+        token: ${{ secrets.DFHACK_GITHUB_TOKEN }}
+        path: xml
+    - name: Download DFHack
+      uses: actions/download-artifact@v4
+      with:
+        name: dfhack-symbols-windows64-build
+    - name: Setup steamcmd
+      id: steamcmd
+      uses: CyberAndrii/setup-steamcmd@v1
+    - name: Update DF version string
+      env:
+        STEAM_USERNAME: ${{ secrets.STEAM_USERNAME }}
+        STEAM_CONFIG_VDF: ${{ secrets.STEAM_CONFIG_VDF }}
+        STEAM_DF_TESTING: ${{ secrets.STEAM_DF_TESTING }}
+        STEAM_DF_ADVENTURE_TEST: ${{ secrets.STEAM_DF_ADVENTURE_TEST }}
+      shell: bash
+      run: |
+        mkdir DF_steam
+        echo "$STEAM_CONFIG_VDF" | base64 -d >${{ steps.steamcmd.outputs.directory }}/config/config.vdf
+        echo "DF steam branch: ${{ inputs.df_steam_branch }}"
+        if [ "${{ inputs.df_steam_branch }}" = "default" ]; then
+          BETA_PARAMS=""
+        elif [ "${{ inputs.df_steam_branch }}" = "testing" ]; then
+          BETA_PARAMS="-beta testing -betapassword $STEAM_DF_TESTING"
+        elif [ "${{ inputs.df_steam_branch }}" = "adventure_test" ]; then
+          BETA_PARAMS="-beta adventure_test -betapassword $STEAM_DF_ADVENTURE_TEST"
+        else
+          BETA_PARAMS="-beta ${{ inputs.df_steam_branch }}"
+        fi
+        ${{ steps.steamcmd.outputs.executable }} \
+          +@ShutdownOnFailedCommand 1 \
+          +@sSteamCmdForcePlatformType windows \
+          +force_install_dir $PWD/DF_steam \
+          +login $STEAM_USERNAME \
+          "+app_update 975370 $BETA_PARAMS validate" \
+          +quit
+        tar xjf dfhack-symbols-windows64-build.tar.bz2 -C DF_steam
+        cp xml/symbols.xml DF_steam/hack
+        cd DF_steam
+        "./Dwarf Fortress.exe" &
+        while ! ./dfhack-run.exe lua 'print(scr)' | fgrep 'viewscreen_titlest' 2>/dev/null; do
+          echo "waiting for DF to start"
+          sleep 0.5
+        done
+        df_ver=`./dfhack-run.exe lua 'print(dfhack.gui.getDFViewscreen(true).str_version)'`
+        echo "Found version string: '$df_ver'"
+        sed -i "s/v0.50.0 win64 STEAM/v0.$df_ver win64 STEAM/" ../xml/symbols.xml
+        ./dfhack-run.exe die || true
+    - name: Merge updates
+      shell: bash
+      run: |
+        cd xml
+        if ! git diff --exit-code; then
+          git stash
+          git pull
+          git stash pop
+        fi
+    - name: Commit symbol updates
+      uses: stefanzweifel/git-auto-commit-action@v5
+      with:
+        commit_message: Auto-update Windows DF version in symbols
+        repository: xml
+        commit_user_name: DFHack-Urist via GitHub Actions
+        commit_user_email: [email protected]
+
   update-ref:
     name: Update structures ref
     runs-on: ubuntu-latest
     needs:
     - generate-linux
-    - generate-windows
+    - auto-ver-windows
     if: ${{ ! failure() }}
     steps:
     - name: Clone DFHack
       uses: actions/checkout@v4
       with:
         token: ${{ secrets.DFHACK_GITHUB_TOKEN }}
     - name: Update ref
+      shell: bash
       run: |
         git submodule update --init --no-single-branch library/xml
         cd library/xml
         git checkout ${{ inputs.structures_ref }}
         git pull
+        df_ver=`grep -E 'symbol-table.*STEAM' symbols.xml | head -n1 | sed -r "s/.*name='v0.([^ ]+) .*/\1/"`
+        echo "using DF version: $df_ver"
+        echo "DETECTED_DF_VER=$df_ver" >>$GITHUB_ENV
     - name: Commit ref update
       uses: stefanzweifel/git-auto-commit-action@v5
       with:
@@ -301,4 +413,4 @@ jobs:
       shell: bash
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: gh workflow run steam-deploy.yml -R DFHack/dfhack -r ${{ github.ref }} -f version=${{ inputs.version }} -f steam_branch=${{ inputs.steam_branch }}
+      run: gh workflow run steam-deploy.yml -R DFHack/dfhack -r ${{ github.ref }} -f version=${{ env.DETECTED_DF_VER }} -f steam_branch=${{ inputs.steam_branch }}

.github/workflows/watch-df-steam.yml

@@ -14,32 +14,27 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # dfhack_ref: leave blank if no structures update is desired
-        # structures_ref: leave blank to default to master
-        # steam_branch: leave blank if no steam push is desired
+        # df_steam_branch: which DF Steam branch to watch
         # platform: leave blank to default to all
-        # channel: leave blank to default to all; should usually be steam
+        # structures_ref: leave blank to default to master
+        # dfhack_ref: leave blank if no structures update is desired
+        # steam_branch: leave blank if no DFHack steam push is desired
         include:
           - df_steam_branch: public
-            version: public
           - df_steam_branch: beta
-            version: 51.01-beta
-            channel: steam
-            dfhack_ref: adv-beta
             structures_ref: adv-beta
+            dfhack_ref: adv-beta
             steam_branch: adventure-beta
           - df_steam_branch: testing
-            version: testing
-            channel: steam
-            dfhack_ref: testing
             structures_ref: testing
+            dfhack_ref: testing
     steps:
     - name: Fetch state
       uses: actions/cache/restore@v4
       with:
         path: state
-        key: watchstate-${{ matrix.version }}
-        restore-keys: watchstate-${{ matrix.version }}
+        key: watchstate-${{ matrix.df_steam_branch }}
+        restore-keys: watchstate-${{ matrix.df_steam_branch }}
     - name: Compare branch metadata
       uses: nick-fields/retry@v3
       with:
@@ -77,14 +72,14 @@ jobs:
           -R DFHack/dfhack \
           -r ${{ matrix.dfhack_ref }} \
           -f structures_ref=${{ matrix.structures_ref }} \
-          -f version=${{ matrix.version }} \
+          -f version=auto \
           -f platform=${{ matrix.platform }} \
-          -f channel=${{ matrix.channel }} \
+          -f channel=steam \
           -f df_steam_branch=${{ matrix.df_steam_branch }} \
           -f steam_branch=${{ matrix.steam_branch }}
     - name: Save state
       uses: actions/cache/save@v4
       if: env.TIMESTAMP
       with:
         path: state
-        key: watchstate-${{ matrix.version }}-${{ env.TIMESTAMP }}
+        key: watchstate-${{ matrix.df_steam_branch }}-${{ env.TIMESTAMP }}

CMakeLists.txt

@@ -56,7 +56,7 @@ endif()
 
 if(WIN32)
     set(MSVC_MIN_VER 1930)
-    set(MSVC_MAX_VER 1941)
+    set(MSVC_MAX_VER 1942)
     if(NOT MSVC)
         message(SEND_ERROR "No MSVC found! MSVC 2022 version ${MSVC_MIN_VER} to ${MSVC_MAX_VER} is required.")
     elseif((MSVC_VERSION LESS MSVC_MIN_VER) OR (MSVC_VERSION GREATER MSVC_MAX_VER))

build/build-win64-from-linux.sh

@@ -32,9 +32,7 @@ if [[ $(id -u) -eq 0 ]]; then
     chown -R $builder_uid win64-cross
 fi
 
-# Assumes you built a container image called dfhack-build-msvc from
-# https://github.com/BenLubar/build-env/tree/master/msvc, see
-# docs/dev/compile/Compile.rst.
+# Pulls the MSVC build env container from the GitHub registry
 #
 # NOTE: win64-cross is mounted in /src/build due to the hardcoded `cmake ..` in
 # the Dockerfile
@@ -44,7 +42,7 @@ if ! docker run --rm -i -v "$srcdir":/src -v "$srcdir/build/win64-cross/":/src/b
     -e steam_username \
     -e steam_password \
     --name dfhack-win \
-    ghcr.io/dfhack/build-env:msvc \
+    ghcr.io/dfhack/build-env:master \
     bash -c "cd /src/build && dfhack-configure windows 64 Release -DCMAKE_INSTALL_PREFIX=/src/build/output -DBUILD_DOCS=1 $CMAKE_EXTRA_ARGS && dfhack-make -j$jobs install" \
     ; then
     echo

docs/changelog.txt

@@ -52,8 +52,8 @@ Template for new versions:
 # Future
 
 ## New Tools
-- `infinite-sky`: (reinstated, renamed from ``infiniteSky``) tool for automatically creating new z-levels of sky to build in
-- `forceequip`: (reinstated) Forcibly move items into a unit's inventory
+- `infinite-sky`: (reinstated, renamed from ``infiniteSky``) automatically create new z-levels of sky to build in
+- `forceequip`: (reinstated) forcibly move items into a unit's inventory
 
 ## New Features
 - `tweak`: ``realistic-melting``: change melting return for inorganic armor parts, shields, weapons, trap components and tools to stop smelters from creating metal, bring melt return for adamantine in line with other metals to ~95% of forging cost. wear reduces melt return by 10% per level
@@ -63,11 +63,10 @@ Template for new versions:
 - `autobutcher`: don't run a scanning and marking cycle on the first tick of a fortress to allow for all custom configuration to be set first
 - `nestboxes`: don't consider eggs to be infertile just because the mother has left the nest; eggs can still hatch in this situation
 - `timestream`: adjust the incubation counter on fertile eggs so they hatch at the expected time
-- `timestream`: fix potential crash in birthday tracking
 - `logistics`: don't ignore rotten items when applying stockpile logistics operations (e.g. autodump, autoclaim, etc.)
 
 ## Misc Improvements
-- DFHack startup now verifies that its idea of the size of certain DF global objects are the same as what DF says they are, and refuses to start if there is a mismatch
+- DFHack now verifies that critical DF data structures have known sizes and refuses to start if there is a mismatch
 - DFHack text edit fields now delete the character at the cursor when you hit the Delete key
 - DFHack text edit fields now move the cursor by one word left or right with Ctrl-Left and Ctrl-Right
 - DFHack text edit fields now move the cursor to the beginning or end of the line with Home and End
@@ -76,18 +75,17 @@ Template for new versions:
 - `preserve-rooms`: automatically release room reservations for captured squad members. we were kidding ourselves with our optimistic kept reservations. they're unlikely to come back : ((
 - `buildingplan`: add value info to item selection dialog (effectively ungrouping items with different values) and add sorting by value
 - `timestream`: reduce CPU utilization
+- `fix/occupancy`: additionally handle the case where tile occupancy doesn't reflect the actual existence of a building
 
 ## Documentation
 - Dreamfort: add link to Dreamfort tutorial youtube series: https://www.youtube.com/playlist?list=PLzXx9JcB9oXxmrtkO1y8ZXzBCFEZrKxve
 - The error message that comes up if there is a version mismatch between DF and the installed DFHack now informs you which DF versions are supported by the installed version of DFHack
 
 ## API
-
 - ``DFHack::Units``: new function ``setPathGoal``
 - ``Units::setAutomaticProfessions``: bay12-provided entry point to assign labors based on work details
 
 ## Lua
-
 - ``dfhack.units``: new function ``setPathGoal``
 - ``widgets.TabBar``: updated to allow for horizontal scrolling of tabs when there are too many to fit in the available space
 

docs/dev/compile/Compile.rst

@@ -303,7 +303,7 @@ Step 1: prepare a build container
 On your Linux host, install and run the docker daemon and then run these commands::
 
     xhost +local:root
-    docker run -it --env="DISPLAY" --env="QT_X11_NO_MITSHM=1" --volume=/tmp/.X11-unix:/tmp/.X11-unix --user buildmaster --name dfhack-win ghcr.io/dfhack/build-env:msvc
+    docker run -it --env="DISPLAY" --env="QT_X11_NO_MITSHM=1" --volume=/tmp/.X11-unix:/tmp/.X11-unix --user buildmaster --name dfhack-win ghcr.io/dfhack/build-env:master
 
 The ``xhost`` command and ``--env`` parameters are there so you can eventually
 run Dwarf Fortress from the container and have it display on your host.

library/Core.cpp

@@ -453,7 +453,7 @@ void get_commands(color_ostream &con, std::vector<std::string> &commands) {
         con.printerr("Failed Lua call to helpdb.get_commands.\n");
     }
 
-    Lua::GetVector(L, commands);
+    Lua::GetVector(L, commands, top + 1);
 }
 
 static bool try_autocomplete(color_ostream &con, const std::string &first, std::string &completed)