external/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2024-11-12 18:04:18 +01:00
Code Issues Projects Releases Packages Wiki Activity

[build] Automated builds and nightly releases (#6220)

Browse source 
Closes #1839
Authored by: Grub4K, bashonly

Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com>
This commit is contained in:
Simon Sawicki 2023-03-03 22:33:12 +05:30 committed by pukkandan
parent d400e261cf
commit 29cb20bd56
No known key found for this signature in database
GPG key ID: 7EEE9E1E817D0A39
9 changed files with 552 additions and 333 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

541
.github/workflows/build.yml vendored
View file

@ -1,393 +1,338 @@
name: Build name: Build Artifacts
on: workflow_dispatch on:
workflow_call:
inputs:
version:
required: true
type: string
channel:
required: false
default: stable
type: string
unix:
default: true
type: boolean
linux_arm:
default: true
type: boolean
macos:
default: true
type: boolean
macos_legacy:
default: true
type: boolean
windows:
default: true
type: boolean
windows32:
default: true
type: boolean
meta_files:
default: true
type: boolean
workflow_dispatch:
inputs:
version:
description: Version tag (YYYY.MM.DD[.REV])
required: true
type: string
channel:
description: Update channel (stable/nightly)
required: true
default: stable
type: string
unix:
description: yt-dlp, yt-dlp.tar.gz, yt-dlp_linux, yt-dlp_linux.zip
default: true
type: boolean
linux_arm:
description: yt-dlp_linux_aarch64, yt-dlp_linux_armv7l
default: true
type: boolean
macos:
description: yt-dlp_macos, yt-dlp_macos.zip
default: true
type: boolean
macos_legacy:
description: yt-dlp_macos_legacy
default: true
type: boolean
windows:
description: yt-dlp.exe, yt-dlp_min.exe, yt-dlp_win.zip
default: true
type: boolean
windows32:
description: yt-dlp_x86.exe
default: true
type: boolean
meta_files:
description: SHA2-256SUMS, SHA2-512SUMS, _update_spec
default: true
type: boolean
permissions: permissions:
contents: read contents: read
jobs: jobs:
prepare: unix:
permissions: if: inputs.unix
contents: write # for push_release
runs-on: ubuntu-latest runs-on: ubuntu-latest
outputs:
version_suffix: ${{ steps.version_suffix.outputs.version_suffix }}
ytdlp_version: ${{ steps.bump_version.outputs.ytdlp_version }}
head_sha: ${{ steps.push_release.outputs.head_sha }}
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: - uses: actions/setup-python@v4
fetch-depth: 0 with:
- uses: actions/setup-python@v4 python-version: "3.10"
with: - uses: conda-incubator/setup-miniconda@v2
python-version: '3.10' with:
- name: Set version suffix
id: version_suffix
env:
PUSH_VERSION_COMMIT: ${{ secrets.PUSH_VERSION_COMMIT }}
if: "env.PUSH_VERSION_COMMIT == ''"
run: echo "version_suffix=$(date -u +"%H%M%S")" >> "$GITHUB_OUTPUT"
- name: Bump version
id: bump_version
run: |
python devscripts/update-version.py ${{ steps.version_suffix.outputs.version_suffix }}
make issuetemplates
- name: Push to release
id: push_release
run: |
git config --global user.name github-actions
git config --global user.email github-actions@example.com
git add -u
git commit -m "[version] update" -m "Created by: ${{ github.event.sender.login }}" -m ":ci skip all :ci run dl"
git push origin --force ${{ github.event.ref }}:release
echo "head_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
- name: Update master
env:
PUSH_VERSION_COMMIT: ${{ secrets.PUSH_VERSION_COMMIT }}
if: "env.PUSH_VERSION_COMMIT != ''"
run: git push origin ${{ github.event.ref }}
build_unix:
needs: prepare
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.10'
- uses: conda-incubator/setup-miniconda@v2
with:
miniforge-variant: Mambaforge miniforge-variant: Mambaforge
use-mamba: true use-mamba: true
channels: conda-forge channels: conda-forge
auto-update-conda: true auto-update-conda: true
activate-environment: '' activate-environment: ""
auto-activate-base: false auto-activate-base: false
- name: Install Requirements - name: Install Requirements
run: | run: |
sudo apt-get -y install zip pandoc man sed sudo apt-get -y install zip pandoc man sed
python -m pip install -U pip setuptools wheel twine python -m pip install -U pip setuptools wheel
python -m pip install -U Pyinstaller -r requirements.txt python -m pip install -U Pyinstaller -r requirements.txt
reqs=$(mktemp) reqs=$(mktemp)
echo -e 'python=3.10.*\npyinstaller' >$reqs echo -e 'python=3.10.*\npyinstaller' >$reqs
sed 's/^brotli.*/brotli-python/' <requirements.txt >>$reqs sed 's/^brotli.*/brotli-python/' <requirements.txt >>$reqs
mamba create -n build --file $reqs mamba create -n build --file $reqs
- name: Prepare - name: Prepare
run: | run: |
python devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }} python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
python devscripts/make_lazy_extractors.py python devscripts/make_lazy_extractors.py
- name: Build Unix platform-independent binary - name: Build Unix platform-independent binary
run: | run: |
make all tar make all tar
- name: Build Unix standalone binary - name: Build Unix standalone binary
shell: bash -l {0} shell: bash -l {0}
run: | run: |
unset LD_LIBRARY_PATH # Harmful; set by setup-python unset LD_LIBRARY_PATH # Harmful; set by setup-python
conda activate build conda activate build
python pyinst.py --onedir python pyinst.py --onedir
(cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .) (cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .)
python pyinst.py python pyinst.py
mv ./dist/yt-dlp_linux ./yt-dlp_linux
mv ./dist/yt-dlp_linux.zip ./yt-dlp_linux.zip
- name: Upload artifacts - name: Upload artifacts
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
path: | path: |
yt-dlp yt-dlp
yt-dlp.tar.gz yt-dlp.tar.gz
dist/yt-dlp_linux yt-dlp_linux
dist/yt-dlp_linux.zip yt-dlp_linux.zip
- name: Build and publish on PyPi linux_arm:
env: if: inputs.linux_arm
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
if: "env.TWINE_PASSWORD != ''"
run: |
rm -rf dist/*
python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update"
python setup.py sdist bdist_wheel
twine upload dist/*
- name: Install SSH private key for Homebrew
env:
BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
if: "env.BREW_TOKEN != ''"
uses: yt-dlp/ssh-agent@v0.5.3
with:
ssh-private-key: ${{ env.BREW_TOKEN }}
- name: Update Homebrew Formulae
env:
BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
if: "env.BREW_TOKEN != ''"
run: |
git clone git@github.com:yt-dlp/homebrew-taps taps/
python devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ needs.prepare.outputs.ytdlp_version }}"
git -C taps/ config user.name github-actions
git -C taps/ config user.email github-actions@example.com
git -C taps/ commit -am 'yt-dlp: ${{ needs.prepare.outputs.ytdlp_version }}'
git -C taps/ push
build_linux_arm:
permissions: permissions:
packages: write # for Creating cache contents: read
packages: write # for creating cache
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: prepare
strategy: strategy:
matrix: matrix:
architecture: architecture:
- armv7 - armv7
- aarch64 - aarch64
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:
path: ./repo path: ./repo
- name: Virtualized Install, Prepare & Build - name: Virtualized Install, Prepare & Build
uses: yt-dlp/run-on-arch-action@v2 uses: yt-dlp/run-on-arch-action@v2
with: with:
githubToken: ${{ github.token }} # To cache image # Ref: https://github.com/uraimo/run-on-arch-action/issues/55
arch: ${{ matrix.architecture }} env: |
distro: ubuntu18.04 # Standalone executable should be built on minimum supported OS GITHUB_WORKFLOW: build
dockerRunArgs: --volume "${PWD}/repo:/repo" githubToken: ${{ github.token }} # To cache image
install: | # Installing Python 3.10 from the Deadsnakes repo raises errors arch: ${{ matrix.architecture }}
apt update distro: ubuntu18.04 # Standalone executable should be built on minimum supported OS
apt -y install zlib1g-dev python3.8 python3.8-dev python3.8-distutils python3-pip dockerRunArgs: --volume "${PWD}/repo:/repo"
python3.8 -m pip install -U pip setuptools wheel install: | # Installing Python 3.10 from the Deadsnakes repo raises errors
# Cannot access requirements.txt from the repo directory at this stage apt update
python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi apt -y install zlib1g-dev python3.8 python3.8-dev python3.8-distutils python3-pip
python3.8 -m pip install -U pip setuptools wheel
# Cannot access requirements.txt from the repo directory at this stage
python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi
run: | run: |
cd repo cd repo
python3.8 -m pip install -U Pyinstaller -r requirements.txt # Cached version may be out of date python3.8 -m pip install -U Pyinstaller -r requirements.txt # Cached version may be out of date
python3.8 devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }} python3.8 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
python3.8 devscripts/make_lazy_extractors.py python3.8 devscripts/make_lazy_extractors.py
python3.8 pyinst.py python3.8 pyinst.py
- name: Upload artifacts - name: Upload artifacts
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
path: | # run-on-arch-action designates armv7l as armv7 path: | # run-on-arch-action designates armv7l as armv7
repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }} repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}
macos:
build_macos: if: inputs.macos
runs-on: macos-11 runs-on: macos-11
needs: prepare
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
# NB: In order to create a universal2 application, the version of python3 in /usr/bin has to be used # NB: In order to create a universal2 application, the version of python3 in /usr/bin has to be used
- name: Install Requirements - name: Install Requirements
run: | run: |
brew install coreutils brew install coreutils
/usr/bin/python3 -m pip install -U --user pip Pyinstaller -r requirements.txt /usr/bin/python3 -m pip install -U --user pip Pyinstaller -r requirements.txt
- name: Prepare - name: Prepare
run: | run: |
/usr/bin/python3 devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }} /usr/bin/python3 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
/usr/bin/python3 devscripts/make_lazy_extractors.py /usr/bin/python3 devscripts/make_lazy_extractors.py
- name: Build - name: Build
run: | run: |
/usr/bin/python3 pyinst.py --target-architecture universal2 --onedir /usr/bin/python3 pyinst.py --target-architecture universal2 --onedir
(cd ./dist/yt-dlp_macos && zip -r ../yt-dlp_macos.zip .) (cd ./dist/yt-dlp_macos && zip -r ../yt-dlp_macos.zip .)
/usr/bin/python3 pyinst.py --target-architecture universal2 /usr/bin/python3 pyinst.py --target-architecture universal2
- name: Upload artifacts - name: Upload artifacts
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
path: | path: |
dist/yt-dlp_macos dist/yt-dlp_macos
dist/yt-dlp_macos.zip dist/yt-dlp_macos.zip
macos_legacy:
build_macos_legacy: if: inputs.macos_legacy
runs-on: macos-latest runs-on: macos-latest
needs: prepare
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- name: Install Python - name: Install Python
# We need the official Python, because the GA ones only support newer macOS versions # We need the official Python, because the GA ones only support newer macOS versions
env: env:
PYTHON_VERSION: 3.10.5 PYTHON_VERSION: 3.10.5
MACOSX_DEPLOYMENT_TARGET: 10.9 # Used up by the Python build tools MACOSX_DEPLOYMENT_TARGET: 10.9 # Used up by the Python build tools
run: | run: |
# Hack to get the latest patch version. Uncomment if needed # Hack to get the latest patch version. Uncomment if needed
#brew install python@3.10 #brew install python@3.10
#export PYTHON_VERSION=$( $(brew --prefix)/opt/python@3.10/bin/python3 --version | cut -d ' ' -f 2 ) #export PYTHON_VERSION=$( $(brew --prefix)/opt/python@3.10/bin/python3 --version | cut -d ' ' -f 2 )
curl https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-macos11.pkg -o "python.pkg" curl https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-macos11.pkg -o "python.pkg"
sudo installer -pkg python.pkg -target / sudo installer -pkg python.pkg -target /
python3 --version python3 --version
- name: Install Requirements - name: Install Requirements
run: | run: |
brew install coreutils brew install coreutils
python3 -m pip install -U --user pip Pyinstaller -r requirements.txt python3 -m pip install -U --user pip Pyinstaller -r requirements.txt
- name: Prepare - name: Prepare
run: | run: |
python3 devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }} python3 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
python3 devscripts/make_lazy_extractors.py python3 devscripts/make_lazy_extractors.py
- name: Build - name: Build
run: | run: |
python3 pyinst.py python3 pyinst.py
mv dist/yt-dlp_macos dist/yt-dlp_macos_legacy mv dist/yt-dlp_macos dist/yt-dlp_macos_legacy
- name: Upload artifacts - name: Upload artifacts
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
path: | path: |
dist/yt-dlp_macos_legacy dist/yt-dlp_macos_legacy
windows:
build_windows: if: inputs.windows
runs-on: windows-latest runs-on: windows-latest
needs: prepare
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: actions/setup-python@v4 - uses: actions/setup-python@v4
with: # 3.8 is used for Win7 support with: # 3.8 is used for Win7 support
python-version: '3.8' python-version: "3.8"
- name: Install Requirements - name: Install Requirements
run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
python -m pip install -U pip setuptools wheel py2exe python -m pip install -U pip setuptools wheel py2exe
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
- name: Prepare - name: Prepare
run: | run: |
python devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }} python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
python devscripts/make_lazy_extractors.py python devscripts/make_lazy_extractors.py
- name: Build - name: Build
run: | run: |
python setup.py py2exe python setup.py py2exe
Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_min.exe Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_min.exe
python pyinst.py python pyinst.py
python pyinst.py --onedir python pyinst.py --onedir
Compress-Archive -Path ./dist/yt-dlp/* -DestinationPath ./dist/yt-dlp_win.zip Compress-Archive -Path ./dist/yt-dlp/* -DestinationPath ./dist/yt-dlp_win.zip
- name: Upload artifacts - name: Upload artifacts
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
path: | path: |
dist/yt-dlp.exe dist/yt-dlp.exe
dist/yt-dlp_min.exe dist/yt-dlp_min.exe
dist/yt-dlp_win.zip dist/yt-dlp_win.zip
windows32:
build_windows32: if: inputs.windows32
runs-on: windows-latest runs-on: windows-latest
needs: prepare
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: actions/setup-python@v4 - uses: actions/setup-python@v4
with: # 3.7 is used for Vista support. See https://github.com/yt-dlp/yt-dlp/issues/390 with: # 3.7 is used for Vista support. See https://github.com/yt-dlp/yt-dlp/issues/390
python-version: '3.7' python-version: "3.7"
architecture: 'x86' architecture: "x86"
- name: Install Requirements - name: Install Requirements
run: | run: |
python -m pip install -U pip setuptools wheel python -m pip install -U pip setuptools wheel
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
- name: Prepare - name: Prepare
run: | run: |
python devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }} python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
python devscripts/make_lazy_extractors.py python devscripts/make_lazy_extractors.py
- name: Build - name: Build
run: | run: |
python pyinst.py python pyinst.py
- name: Upload artifacts - name: Upload artifacts
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
path: | path: |
dist/yt-dlp_x86.exe dist/yt-dlp_x86.exe
meta_files:
publish_release: if: inputs.meta_files && always()
permissions: needs:
contents: write # for action-gh-release - unix
- linux_arm
- macos
- macos_legacy
- windows
- windows32
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: [prepare, build_unix, build_linux_arm, build_windows, build_windows32, build_macos, build_macos_legacy]
steps: steps:
- uses: actions/checkout@v3 - uses: actions/download-artifact@v3
- uses: actions/download-artifact@v3
- name: Get Changelog - name: Make SHA2-SUMS files
run: | run: |
changelog=$(grep -oPz '(?s)(?<=### ${{ needs.prepare.outputs.ytdlp_version }}\n{2}).+?(?=\n{2,3}###)' Changelog.md) || true cd ./artifact/
echo "changelog<<EOF" >> $GITHUB_ENV sha256sum * > ../SHA2-256SUMS
echo "$changelog" >> $GITHUB_ENV sha512sum * > ../SHA2-512SUMS
echo "EOF" >> $GITHUB_ENV
- name: Make Update spec
run: |
echo "# This file is used for regulating self-update" >> _update_spec
echo "lock 2022.07.18 .+ Python 3.6" >> _update_spec
- name: Make SHA2-SUMS files
run: |
sha256sum artifact/yt-dlp | awk '{print $1 " yt-dlp"}' >> SHA2-256SUMS
sha256sum artifact/yt-dlp.tar.gz | awk '{print $1 " yt-dlp.tar.gz"}' >> SHA2-256SUMS
sha256sum artifact/yt-dlp.exe | awk '{print $1 " yt-dlp.exe"}' >> SHA2-256SUMS
sha256sum artifact/yt-dlp_win.zip | awk '{print $1 " yt-dlp_win.zip"}' >> SHA2-256SUMS
sha256sum artifact/yt-dlp_min.exe | awk '{print $1 " yt-dlp_min.exe"}' >> SHA2-256SUMS
sha256sum artifact/yt-dlp_x86.exe | awk '{print $1 " yt-dlp_x86.exe"}' >> SHA2-256SUMS
sha256sum artifact/yt-dlp_macos | awk '{print $1 " yt-dlp_macos"}' >> SHA2-256SUMS
sha256sum artifact/yt-dlp_macos.zip | awk '{print $1 " yt-dlp_macos.zip"}' >> SHA2-256SUMS
sha256sum artifact/yt-dlp_macos_legacy | awk '{print $1 " yt-dlp_macos_legacy"}' >> SHA2-256SUMS
sha256sum artifact/yt-dlp_linux_armv7l | awk '{print $1 " yt-dlp_linux_armv7l"}' >> SHA2-256SUMS
sha256sum artifact/yt-dlp_linux_aarch64 | awk '{print $1 " yt-dlp_linux_aarch64"}' >> SHA2-256SUMS
sha256sum artifact/dist/yt-dlp_linux | awk '{print $1 " yt-dlp_linux"}' >> SHA2-256SUMS
sha256sum artifact/dist/yt-dlp_linux.zip | awk '{print $1 " yt-dlp_linux.zip"}' >> SHA2-256SUMS
sha512sum artifact/yt-dlp | awk '{print $1 " yt-dlp"}' >> SHA2-512SUMS
sha512sum artifact/yt-dlp.tar.gz | awk '{print $1 " yt-dlp.tar.gz"}' >> SHA2-512SUMS
sha512sum artifact/yt-dlp.exe | awk '{print $1 " yt-dlp.exe"}' >> SHA2-512SUMS
sha512sum artifact/yt-dlp_win.zip | awk '{print $1 " yt-dlp_win.zip"}' >> SHA2-512SUMS
sha512sum artifact/yt-dlp_min.exe | awk '{print $1 " yt-dlp_min.exe"}' >> SHA2-512SUMS
sha512sum artifact/yt-dlp_x86.exe | awk '{print $1 " yt-dlp_x86.exe"}' >> SHA2-512SUMS
sha512sum artifact/yt-dlp_macos | awk '{print $1 " yt-dlp_macos"}' >> SHA2-512SUMS
sha512sum artifact/yt-dlp_macos.zip | awk '{print $1 " yt-dlp_macos.zip"}' >> SHA2-512SUMS
sha512sum artifact/yt-dlp_macos_legacy | awk '{print $1 " yt-dlp_macos_legacy"}' >> SHA2-512SUMS
sha512sum artifact/yt-dlp_linux_armv7l | awk '{print $1 " yt-dlp_linux_armv7l"}' >> SHA2-512SUMS
sha512sum artifact/yt-dlp_linux_aarch64 | awk '{print $1 " yt-dlp_linux_aarch64"}' >> SHA2-512SUMS
sha512sum artifact/dist/yt-dlp_linux | awk '{print $1 " yt-dlp_linux"}' >> SHA2-512SUMS
sha512sum artifact/dist/yt-dlp_linux.zip | awk '{print $1 " yt-dlp_linux.zip"}' >> SHA2-512SUMS
- name: Publish Release - name: Make Update spec
uses: yt-dlp/action-gh-release@v1 run: |
with: cat >> _update_spec << EOF
tag_name: ${{ needs.prepare.outputs.ytdlp_version }} # This file is used for regulating self-update
name: yt-dlp ${{ needs.prepare.outputs.ytdlp_version }} lock 2022.08.18.36 .+ Python 3.6
target_commitish: ${{ needs.prepare.outputs.head_sha }} EOF
body: |
#### [A description of the various files]((https://github.com/yt-dlp/yt-dlp#release-files)) are in the README
--- - name: Upload artifacts
<details open><summary><h3>Changelog</summary> uses: actions/upload-artifact@v3
<p> with:
path: |
${{ env.changelog }} SHA*SUMS*
_update_spec
</p>
</details>
files: |
SHA2-256SUMS
SHA2-512SUMS
artifact/yt-dlp
artifact/yt-dlp.tar.gz
artifact/yt-dlp.exe
artifact/yt-dlp_win.zip
artifact/yt-dlp_min.exe
artifact/yt-dlp_x86.exe
artifact/yt-dlp_macos
artifact/yt-dlp_macos.zip
artifact/yt-dlp_macos_legacy
artifact/yt-dlp_linux_armv7l
artifact/yt-dlp_linux_aarch64
artifact/dist/yt-dlp_linux
artifact/dist/yt-dlp_linux.zip
_update_spec

80
.github/workflows/publish.yml vendored Normal file
View file

@ -0,0 +1,80 @@
name: Publish
on:
workflow_call:
inputs:
nightly:
default: false
required: false
type: boolean
version:
required: true
type: string
target_commitish:
required: true
type: string
secrets:
ARCHIVE_REPO_TOKEN:
required: false
permissions:
contents: write
jobs:
publish:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: actions/download-artifact@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Generate release notes
run: |
cat >> ./RELEASE_NOTES << EOF
#### A description of the various files are in the [README](https://github.com/yt-dlp/yt-dlp#release-files)
---
<details><summary><h3>Changelog</h3></summary>
$(python ./devscripts/make_changelog.py -vv)
</details>
EOF
echo "**This is an automated nightly pre-release build**" >> ./PRERELEASE_NOTES
cat ./RELEASE_NOTES >> ./PRERELEASE_NOTES
echo "Generated from: https://github.com/${{ github.repository }}/commit/${{ inputs.target_commitish }}" >> ./ARCHIVE_NOTES
cat ./RELEASE_NOTES >> ./ARCHIVE_NOTES
- name: Archive nightly release
env:
GH_TOKEN: ${{ secrets.ARCHIVE_REPO_TOKEN }}
GH_REPO: ${{ vars.ARCHIVE_REPO }}
if: |
inputs.nightly && env.GH_TOKEN != '' && env.GH_REPO != ''
run: |
gh release create \
--notes-file ARCHIVE_NOTES \
--title "Build ${{ inputs.version }}" \
${{ inputs.version }} \
artifact/*
- name: Prune old nightly release
if: inputs.nightly
env:
GH_TOKEN: ${{ github.token }}
run: |
gh release delete --yes --cleanup-tag "nightly" || true
git tag --delete "nightly" || true
sleep 5 # Enough time to cover deletion race condition
- name: Publish release${{ inputs.nightly && ' (nightly)' || '' }}
env:
GH_TOKEN: ${{ github.token }}
run: |
gh release create \
--notes-file ${{ inputs.nightly && 'PRE' || '' }}RELEASE_NOTES \
--target ${{ inputs.target_commitish }} \
--title "yt-dlp ${{ inputs.nightly && 'nightly ' || '' }}${{ inputs.version }}" \
${{ inputs.nightly && '--prerelease "nightly"' || inputs.version }} \
artifact/*

49
.github/workflows/release-nightly.yml vendored Normal file
View file

@ -0,0 +1,49 @@
name: Release (nightly)
on:
push:
branches:
- master
paths:
- "**.py"
- "!yt_dlp/version.py"
concurrency:
group: release-nightly
cancel-in-progress: true
permissions:
contents: read
jobs:
prepare:
if: vars.BUILD_NIGHTLY != ''
runs-on: ubuntu-latest
outputs:
version: ${{ steps.get_version.outputs.version }}
steps:
- uses: actions/checkout@v3
- name: Get version
id: get_version
run: |
python devscripts/update-version.py "$(date -u +"%H%M%S")" | grep -Po "version=\d+(\.\d+){3}" >> "$GITHUB_OUTPUT"
build:
needs: prepare
uses: ./.github/workflows/build.yml
with:
version: ${{ needs.prepare.outputs.version }}
channel: nightly
permissions:
contents: read
packages: write # For package cache
publish:
needs: [prepare, build]
uses: ./.github/workflows/publish.yml
secrets:
ARCHIVE_REPO_TOKEN: ${{ secrets.ARCHIVE_REPO_TOKEN }}
permissions:
contents: write
with:
nightly: true
version: ${{ needs.prepare.outputs.version }}
target_commitish: ${{ github.sha }}

125
.github/workflows/release.yml vendored Normal file
View file

@ -0,0 +1,125 @@
name: Release
on: workflow_dispatch
permissions:
contents: read
jobs:
prepare:
permissions:
contents: write
runs-on: ubuntu-latest
outputs:
version: ${{ steps.update_version.outputs.version }}
head_sha: ${{ steps.push_release.outputs.head_sha }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Update version
id: update_version
run: |
python devscripts/update-version.py ${{ vars.PUSH_VERSION_COMMIT == '' && '"$(date -u +"%H%M%S")"' || '' }} | \
grep -Po "version=\d+\.\d+\.\d+(\.\d+)?" >> "$GITHUB_OUTPUT"
- name: Update documentation
run: |
make doc
sed '/### /Q' Changelog.md >> ./CHANGELOG
echo '### ${{ steps.update_version.outputs.version }}' >> ./CHANGELOG
python ./devscripts/make_changelog.py -vv -c >> ./CHANGELOG
echo >> ./CHANGELOG
grep -Poz '(?s)### \d+\.\d+\.\d+.+' 'Changelog.md' | head -n -1 >> ./CHANGELOG
cat ./CHANGELOG > Changelog.md
- name: Push to release
id: push_release
run: |
git config --global user.name github-actions
git config --global user.email github-actions@example.com
git add -u
git commit -m "Release ${{ steps.update_version.outputs.version }}" \
-m "Created by: ${{ github.event.sender.login }}" -m ":ci skip all :ci run dl"
git push origin --force ${{ github.event.ref }}:release
echo "head_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
- name: Update master
if: vars.PUSH_VERSION_COMMIT != ''
run: git push origin ${{ github.event.ref }}
publish_pypi_homebrew:
needs: prepare
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Install Requirements
run: |
python -m pip install -U pip setuptools wheel twine
python -m pip install -U -r requirements.txt
- name: Prepare
run: |
python devscripts/update-version.py ${{ needs.prepare.outputs.version }}
python devscripts/make_lazy_extractors.py
- name: Build and publish on PyPI
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
if: env.TWINE_PASSWORD != ''
run: |
rm -rf dist/*
python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update"
python setup.py sdist bdist_wheel
twine upload dist/*
- name: Checkout Homebrew repository
env:
BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
if: env.BREW_TOKEN != '' && env.PYPI_TOKEN != ''
uses: actions/checkout@v3
with:
repository: yt-dlp/homebrew-taps
path: taps
ssh-key: ${{ secrets.BREW_TOKEN }}
- name: Update Homebrew Formulae
env:
BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
if: env.BREW_TOKEN != '' && env.PYPI_TOKEN != ''
run: |
python devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ needs.prepare.outputs.version }}"
git -C taps/ config user.name github-actions
git -C taps/ config user.email github-actions@example.com
git -C taps/ commit -am 'yt-dlp: ${{ needs.prepare.outputs.version }}'
git -C taps/ push
build:
needs: prepare
uses: ./.github/workflows/build.yml
with:
version: ${{ needs.prepare.outputs.version }}
permissions:
contents: read
packages: write # For package cache
publish:
needs: [prepare, build]
uses: ./.github/workflows/publish.yml
permissions:
contents: write
with:
version: ${{ needs.prepare.outputs.version }}
target_commitish: ${{ needs.prepare.outputs.head_sha }}

8
Changelog.md
View file

@ -1,13 +1,7 @@
# Changelog # Changelog
<!-- <!--
# Instuctions for creating release # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
* Run `make doc`
* Update Changelog.md and CONTRIBUTORS
* Change "Based on ytdl" version in Readme.md if needed
* Commit as `Release <version>` and push to master
* Dispatch the workflow https://github.com/yt-dlp/yt-dlp/actions/workflows/build.yml on master
--> -->
### 2023.02.17 ### 2023.02.17

8
README.md
View file

@ -318,7 +318,8 @@ ### Related scripts
Note: See their `--help` for more info. Note: See their `--help` for more info.
You can also fork the project on GitHub and run your fork's [build workflow](.github/workflows/build.yml) to automatically build a full release ### Forking the project
If you fork the project on GitHub, you can run your fork's [build workflow](.github/workflows/build.yml) to automatically build the selected version(s) as artifacts. Alternatively, you can run the [release workflow](.github/workflows/release.yml) or enable the [nightly workflow](.github/workflows/release-nightly.yml) to create full (pre-)releases.
# USAGE AND OPTIONS # USAGE AND OPTIONS
@ -460,9 +461,8 @@ ## Video Selection:
--date DATE Download only videos uploaded on this date. --date DATE Download only videos uploaded on this date.
The date can be "YYYYMMDD" or in the format The date can be "YYYYMMDD" or in the format
[now|today|yesterday][-N[day|week|month|year]]. [now|today|yesterday][-N[day|week|month|year]].
E.g. "--date today-2weeks" downloads E.g. "--date today-2weeks" downloads only
only videos uploaded on the same day two videos uploaded on the same day two weeks ago
weeks ago
--datebefore DATE Download only videos uploaded on or before --datebefore DATE Download only videos uploaded on or before
this date. The date formats accepted is the this date. The date formats accepted is the
same as --date same as --date

22
devscripts/make_readme.py
View file

@ -45,33 +45,43 @@ def apply_patch(text, patch):
delim = f'\n{" " * switch_col_width}' delim = f'\n{" " * switch_col_width}'
PATCHES = ( PATCHES = (
( # Standardize update message ( # Standardize `--update` message
r'(?m)^( -U, --update\s+).+(\n \s.+)*$', r'(?m)^( -U, --update\s+).+(\n \s.+)*$',
r'\1Update this program to the latest version', r'\1Update this program to the latest version',
), ),
( # Headings ( # Headings
r'(?m)^ (\w.+\n)( (?=\w))?', r'(?m)^ (\w.+\n)( (?=\w))?',
r'## \1' r'## \1'
), ),
( # Do not split URLs ( # Fixup `--date` formatting
rf'(?m)( --date DATE.+({delim}[^\[]+)*)\[.+({delim}.+)*$',
(rf'\1[now|today|yesterday][-N[day|week|month|year]].{delim}'
f'E.g. "--date today-2weeks" downloads only{delim}'
'videos uploaded on the same day two weeks ago'),
),
( # Do not split URLs
rf'({delim[:-1]})? (?P<label>\[\S+\] )?(?P<url>https?({delim})?:({delim})?/({delim})?/(({delim})?\S+)+)\s', rf'({delim[:-1]})? (?P<label>\[\S+\] )?(?P<url>https?({delim})?:({delim})?/({delim})?/(({delim})?\S+)+)\s',
lambda mobj: ''.join((delim, mobj.group('label') or '', re.sub(r'\s+', '', mobj.group('url')), '\n')) lambda mobj: ''.join((delim, mobj.group('label') or '', re.sub(r'\s+', '', mobj.group('url')), '\n'))
), ),
( # Do not split "words" ( # Do not split "words"
rf'(?m)({delim}\S+)+$', rf'(?m)({delim}\S+)+$',
lambda mobj: ''.join((delim, mobj.group(0).replace(delim, ''))) lambda mobj: ''.join((delim, mobj.group(0).replace(delim, '')))
), ),
( # Allow overshooting last line ( # Allow overshooting last line
rf'(?m)^(?P<prev>.+)${delim}(?P<current>.+)$(?!{delim})', rf'(?m)^(?P<prev>.+)${delim}(?P<current>.+)$(?!{delim})',
lambda mobj: (mobj.group().replace(delim, ' ') lambda mobj: (mobj.group().replace(delim, ' ')
if len(mobj.group()) - len(delim) + 1 <= max_width + ALLOWED_OVERSHOOT if len(mobj.group()) - len(delim) + 1 <= max_width + ALLOWED_OVERSHOOT
else mobj.group()) else mobj.group())
), ),
( # Avoid newline when a space is available b/w switch and description ( # Avoid newline when a space is available b/w switch and description
DISABLE_PATCH, # This creates issues with prepare_manpage DISABLE_PATCH, # This creates issues with prepare_manpage
r'(?m)^(\s{4}-.{%d})(%s)' % (switch_col_width - 6, delim), r'(?m)^(\s{4}-.{%d})(%s)' % (switch_col_width - 6, delim),
r'\1 ' r'\1 '
), ),
( # Replace brackets with a Markdown link
r'SponsorBlock API \((http.+)\)',
r'[SponsorBlock API](\1)'
),
) )
readme = read_file(README_FILE) readme = read_file(README_FILE)

46
devscripts/update-version.py
View file

@ -7,6 +7,7 @@
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import argparse
import contextlib import contextlib
import subprocess import subprocess
import sys import sys
@ -15,8 +16,9 @@
from devscripts.utils import read_version, write_file from devscripts.utils import read_version, write_file
def get_new_version(revision): def get_new_version(version, revision):
version = datetime.utcnow().strftime('%Y.%m.%d') if not version:
version = datetime.utcnow().strftime('%Y.%m.%d')
if revision: if revision:
assert revision.isdigit(), 'Revision must be a number' assert revision.isdigit(), 'Revision must be a number'
@ -30,27 +32,41 @@ def get_new_version(revision):
def get_git_head(): def get_git_head():
with contextlib.suppress(Exception): with contextlib.suppress(Exception):
sp = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'], stdout=subprocess.PIPE) return subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip() or None
return sp.communicate()[0].decode().strip() or None
VERSION = get_new_version((sys.argv + [''])[1]) VERSION_TEMPLATE = '''\
GIT_HEAD = get_git_head()
VERSION_FILE = f'''\
# Autogenerated by devscripts/update-version.py # Autogenerated by devscripts/update-version.py
__version__ = {VERSION!r} __version__ = {version!r}
RELEASE_GIT_HEAD = {GIT_HEAD!r} RELEASE_GIT_HEAD = {git_head!r}
VARIANT = None VARIANT = None
UPDATE_HINT = None UPDATE_HINT = None
CHANNEL = "{channel!r}"
''' '''
write_file('yt_dlp/version.py', VERSION_FILE) if __name__ == '__main__':
github_output = os.getenv('GITHUB_OUTPUT') parser = argparse.ArgumentParser(description='Update the version.py file')
if github_output: parser.add_argument(
write_file(github_output, f'ytdlp_version={VERSION}\n', 'a') '-c', '--channel', choices=['stable', 'nightly'], default='stable',
print(f'\nVersion = {VERSION}, Git HEAD = {GIT_HEAD}') help='Select update channel (default: %(default)s)')
parser.add_argument(
'-o', '--output', default='yt_dlp/version.py',
help='The output file to write to (default: %(default)s)')
parser.add_argument(
'version', nargs='?', default=None,
help='A version or revision to use instead of generating one')
args = parser.parse_args()
git_head = get_git_head()
version = (
args.version if args.version and '.' in args.version
else get_new_version(None, args.version))
write_file(args.output, VERSION_TEMPLATE.format(
version=version, git_head=git_head, channel=args.channel))
print(f'version={version} ({args.channel}), head={git_head}')

6
yt_dlp/YoutubeDL.py
View file

@ -150,7 +150,7 @@
write_json_file, write_json_file,
write_string, write_string,
) )
from .version import RELEASE_GIT_HEAD, VARIANT, __version__ from .version import CHANNEL, RELEASE_GIT_HEAD, VARIANT, __version__
if compat_os_name == 'nt': if compat_os_name == 'nt':
import ctypes import ctypes
@ -3768,8 +3768,8 @@ def get_encoding(stream):
klass = type(self) klass = type(self)
write_debug(join_nonempty( write_debug(join_nonempty(
f'{"yt-dlp" if REPOSITORY == "yt-dlp/yt-dlp" else REPOSITORY} version', f'{"yt-dlp" if REPOSITORY == "yt-dlp/yt-dlp" else REPOSITORY} version',
__version__, __version__ + {'stable': '', 'nightly': '*'}.get(CHANNEL, f' <{CHANNEL}>'),
f'[{RELEASE_GIT_HEAD}]' if RELEASE_GIT_HEAD else '', f'[{RELEASE_GIT_HEAD[:9]}]' if RELEASE_GIT_HEAD else '',
'' if source == 'unknown' else f'({source})', '' if source == 'unknown' else f'({source})',
'' if _IN_CLI else 'API' if klass == YoutubeDL else f'API:{self.__module__}.{klass.__qualname__}', '' if _IN_CLI else 'API' if klass == YoutubeDL else f'API:{self.__module__}.{klass.__qualname__}',
delim=' ')) delim=' '))