Update 2025-04-12_09:56:11
This commit is contained in:
oib
54
2mp3cudaRestoration.sh
Executable file
54
2mp3cudaRestoration.sh
Executable file
@ -0,0 +1,54 @@
|
||||
#!/bin/bash
|
||||
# Script Version: 03
|
||||
# Description: Simple script to convert audio files to MP3 (192 kbps) using CUDA for acceleration.
|
||||
|
||||
# Set variables
|
||||
# ========
|
||||
INPUT_DIR="$(pwd)"
|
||||
OUTPUT_DIR="$INPUT_DIR/output"
|
||||
BITRATE="192k"
|
||||
|
||||
# Functions
|
||||
# ========
|
||||
convert_to_mp3() {
|
||||
local INPUT_FILE="$1"
|
||||
local OUTPUT_FILE="$2"
|
||||
|
||||
# Convert to MP3 with FFmpeg
|
||||
ffmpeg -hwaccel cuda -i "$INPUT_FILE" -c:a libmp3lame -b:a "$BITRATE" "$OUTPUT_FILE" -y || return 1
|
||||
}
|
||||
|
||||
# Main Process
|
||||
# ========
|
||||
echo "Starting audio conversion process..."
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
SUCCESS_COUNT=0
|
||||
TOTAL_COUNT=0
|
||||
|
||||
shopt -s nullglob
|
||||
for FILE in "$INPUT_DIR"/*; do
|
||||
if [[ -f "$FILE" ]]; then
|
||||
BASENAME=$(basename "$FILE")
|
||||
EXTENSION="${BASENAME##*.}"
|
||||
|
||||
# Skip unsupported extensions
|
||||
if ! [[ "$EXTENSION" =~ ^(wav|flac|opus|m4a|mp3)$ ]]; then
|
||||
echo "Skipping unsupported file: $FILE"
|
||||
continue
|
||||
fi
|
||||
|
||||
OUTPUT_FILE="$OUTPUT_DIR/${BASENAME%.*}.mp3"
|
||||
|
||||
echo "Converting $FILE to $OUTPUT_FILE"
|
||||
if convert_to_mp3 "$FILE" "$OUTPUT_FILE"; then
|
||||
SUCCESS_COUNT=$((SUCCESS_COUNT + 1))
|
||||
fi
|
||||
TOTAL_COUNT=$((TOTAL_COUNT + 1))
|
||||
fi
|
||||
|
||||
done
|
||||
shopt -u nullglob
|
||||
|
||||
echo "Audio conversion process completed. Success: $SUCCESS_COUNT/$TOTAL_COUNT"
|
||||
|
||||
46
calculate_bitrate.sh
Executable file
46
calculate_bitrate.sh
Executable file
@ -0,0 +1,46 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Check if the folder is provided as an argument
|
||||
if [ -z "$1" ]; then
|
||||
echo "Usage: $0 <folder_with_videos>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
VIDEO_FOLDER="$1"
|
||||
OUTPUT_FILE="bitrates.txt"
|
||||
TOTAL_BITRATE=0
|
||||
VIDEO_COUNT=0
|
||||
|
||||
# Clear or create the output file
|
||||
> "$OUTPUT_FILE"
|
||||
|
||||
# Function to get bitrate of a video in Mbps
|
||||
get_bitrate() {
|
||||
local video_file="$1"
|
||||
bitrate_kbps=$(ffprobe -v error -select_streams v:0 -show_entries stream=bit_rate -of default=noprint_wrappers=1:nokey=1 "$video_file" | head -n 1)
|
||||
if [[ "$bitrate_kbps" =~ ^[0-9]+$ ]]; then
|
||||
bitrate_mbps=$(echo "scale=2; $bitrate_kbps / 1000 / 1000" | bc)
|
||||
echo "$bitrate_mbps"
|
||||
else
|
||||
echo "0"
|
||||
fi
|
||||
}
|
||||
|
||||
# Iterate through each video file in the folder
|
||||
for video_file in "$VIDEO_FOLDER"/*; do
|
||||
if [ -f "$video_file" ]; then
|
||||
bitrate=$(get_bitrate "$video_file")
|
||||
echo "File: $video_file - Bitrate: ${bitrate} Mbps" | tee -a "$OUTPUT_FILE"
|
||||
TOTAL_BITRATE=$(echo "$TOTAL_BITRATE + $bitrate" | bc)
|
||||
((VIDEO_COUNT++))
|
||||
fi
|
||||
done
|
||||
|
||||
# Calculate the average bitrate
|
||||
if [ "$VIDEO_COUNT" -gt 0 ]; then
|
||||
AVERAGE_BITRATE=$(echo "scale=2; $TOTAL_BITRATE / $VIDEO_COUNT" | bc)
|
||||
echo "Average Bitrate: $AVERAGE_BITRATE Mbps" | tee -a "$OUTPUT_FILE"
|
||||
else
|
||||
echo "No video files found in the specified folder." | tee -a "$OUTPUT_FILE"
|
||||
fi
|
||||
|
||||
57
clean_local_bin.sh
Executable file
57
clean_local_bin.sh
Executable file
@ -0,0 +1,57 @@
|
||||
#!/bin/bash
|
||||
# Script Version: 0.1
|
||||
# Description: Interactive cleanup for stale Python tool CLI files in ~/.local/bin (at2 workstation)
|
||||
|
||||
# Set variables
|
||||
# ========
|
||||
TARGET_DIR="$HOME/.local/bin"
|
||||
LOG_FILE="$HOME/.clean_local_bin.log"
|
||||
|
||||
# Functions
|
||||
# ========
|
||||
|
||||
# List suspected legacy tools
|
||||
list_legacy_tools() {
|
||||
find "$TARGET_DIR" -maxdepth 1 -type f \( \
|
||||
-name 'futurize' -o -name 'pasteurize' -o -name 'pkginfo' -o -name 'pybabel' \
|
||||
-o -name 'pygmentize' -o -name 'webassets' -o -name 'wheel' \
|
||||
-o -name 'mutagen-*' -o -name 'mid3*' -o -name 'moggsplit' \
|
||||
-o -name 'filetype' -o -name 'normalizer' -o -name 'markdown*' \
|
||||
-o -name 'jsonschema' -o -name 'httpx' -o -name 'openai' \
|
||||
-o -name 'unidecode' -o -name 'netaddr' -o -name 'flask' \
|
||||
-o -name 'pyserial-*' -o -name 'psg*' \
|
||||
\)
|
||||
}
|
||||
|
||||
# Main Process
|
||||
# ========
|
||||
echo "[DEBUG] Scanning $TARGET_DIR for workshop leftovers..."
|
||||
|
||||
list_legacy_tools > /tmp/.local_bin_candidates.txt
|
||||
|
||||
if [[ ! -s /tmp/.local_bin_candidates.txt ]]; then
|
||||
echo "[DEBUG] Nothing found to delete."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "[DEBUG] Found the following candidates:"
|
||||
cat /tmp/.local_bin_candidates.txt
|
||||
|
||||
echo "[DEBUG] Proceed with deletion? (y/n)"
|
||||
read CONFIRM
|
||||
if [[ "$CONFIRM" != "y" ]]; then
|
||||
echo "[DEBUG] Aborted by user"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[DEBUG] Deleting files and logging to $LOG_FILE"
|
||||
while read -r FILE; do
|
||||
echo "[DEBUG] Removing $FILE"
|
||||
echo "$(date) [DELETED] $FILE" >> "$LOG_FILE"
|
||||
rm -v "$FILE"
|
||||
done < /tmp/.local_bin_candidates.txt
|
||||
|
||||
echo "[DEBUG] Cleanup done."
|
||||
|
||||
# EOF
|
||||
|
||||
48
convert2opus.sh
Executable file
48
convert2opus.sh
Executable file
@ -0,0 +1,48 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Number of concurrent ffmpeg processes
|
||||
N=${1:-4}
|
||||
|
||||
# Create a semaphore with a given number of slots
|
||||
semaphore=$(mktemp)
|
||||
exec 3<>$semaphore
|
||||
for ((i=0; i<N; i++)); do
|
||||
echo >&3
|
||||
done
|
||||
|
||||
# Debugging function
|
||||
debug_log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $@"
|
||||
}
|
||||
|
||||
# Run ffmpeg command with semaphore locking and delete .wav file on success
|
||||
run_ffmpeg() {
|
||||
local file=$1
|
||||
local outfile="${file/%.wav/.opus}"
|
||||
|
||||
# Wait for a slot to become available
|
||||
read -u 3
|
||||
debug_log "Starting conversion: $file"
|
||||
|
||||
if ffmpeg -i "$file" -y -c:a libopus -vbr on -compression_level 5 -ar 48000 -application audio "$outfile"; then
|
||||
debug_log "Completed conversion: $file"
|
||||
debug_log "Deleting original file: $file"
|
||||
rm "$file"
|
||||
else
|
||||
debug_log "Failed conversion: $file"
|
||||
fi
|
||||
|
||||
# Release the slot
|
||||
echo >&3
|
||||
}
|
||||
|
||||
export -f run_ffmpeg
|
||||
export -f debug_log
|
||||
|
||||
# Find all .wav files and convert them to .opus in parallel, respecting semaphore, and delete .wav files on success
|
||||
find . -maxdepth 1 -type f -name '*.wav' -print0 | xargs -0 -I {} -P $N bash -c 'run_ffmpeg "$@"' _ {}
|
||||
|
||||
# Cleanup
|
||||
exec 3>&-
|
||||
rm -f $semaphore
|
||||
|
||||
25
dns_health_check.sh
Executable file
25
dns_health_check.sh
Executable file
@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
# Script Version: 02
|
||||
# Description: Periodically verify consistency between ns1, ns2, and Google nameserver DNS records for a specific domain.
|
||||
|
||||
# Set Variables
|
||||
# ========
|
||||
NS1="23.88.113.138"
|
||||
NS2="116.202.112.180"
|
||||
GOOGLE_NS="8.8.8.8"
|
||||
DOMAIN="es1.dynproxy.net"
|
||||
LOG_FILE="/var/log/dns_health_check.log"
|
||||
|
||||
# Main Process
|
||||
# ========
|
||||
IP_NS1=$(dig @$NS1 $DOMAIN A +short)
|
||||
IP_NS2=$(dig @$NS2 $DOMAIN A +short)
|
||||
IP_GOOGLE=$(dig @$GOOGLE_NS $DOMAIN A +short)
|
||||
|
||||
if [ "$IP_NS1" == "$IP_NS2" ] && [ "$IP_NS1" == "$IP_GOOGLE" ]; then
|
||||
echo "[$(date)] DNS records are consistent across all nameservers: $IP_NS1" >> "$LOG_FILE"
|
||||
else
|
||||
echo "[$(date)] DNS inconsistency detected!" >> "$LOG_FILE"
|
||||
echo "[$(date)] ns1: $IP_NS1, ns2: $IP_NS2, Google: $IP_GOOGLE" >> "$LOG_FILE"
|
||||
fi
|
||||
|
||||
37
fetch_transcript.sh
Executable file
37
fetch_transcript.sh
Executable file
@ -0,0 +1,37 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Check if URL is provided
|
||||
if [ -z "$1" ]; then
|
||||
echo "Usage: $0 <youtube_video_url>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
URL=$1
|
||||
VIDEO_ID=$(echo $URL | grep -o 'v=[^&]*' | cut -d '=' -f 2)
|
||||
if [ -z "$VIDEO_ID" ]; then
|
||||
VIDEO_ID=$(echo $URL | grep -o '[^/]*$')
|
||||
fi
|
||||
OUTPUT_FILE="${VIDEO_ID}.txt"
|
||||
|
||||
# Create a temporary Node.js script
|
||||
cat << EOF > fetch_transcript.js
|
||||
const { YoutubeTranscript } = require('youtube-transcript');
|
||||
const fs = require('fs');
|
||||
|
||||
YoutubeTranscript.fetchTranscript('$VIDEO_ID')
|
||||
.then(transcript => {
|
||||
const transcriptText = transcript.map(item => item.text).join('\\n');
|
||||
fs.writeFileSync('$OUTPUT_FILE', transcriptText);
|
||||
console.log('Transcript saved to $OUTPUT_FILE');
|
||||
})
|
||||
.catch(err => {
|
||||
console.error('Error fetching transcript:', err);
|
||||
});
|
||||
EOF
|
||||
|
||||
# Run the Node.js script
|
||||
node fetch_transcript.js
|
||||
|
||||
# Clean up
|
||||
rm fetch_transcript.js
|
||||
|
||||
76
ffmpeg_video_compress.sh
Executable file
76
ffmpeg_video_compress.sh
Executable file
@ -0,0 +1,76 @@
|
||||
#!/bin/zsh
|
||||
# Script Version: 14
|
||||
# Description: Compress a video using ffmpeg with NVIDIA CUDA for acceleration, aiming for a smaller file size. Push GPU usage and CPU multicore hyperthreading to maximize performance by parallel processing.
|
||||
|
||||
# Set variables
|
||||
# ========
|
||||
INPUT_VIDEO="$1"
|
||||
OUTPUT_VIDEO="${INPUT_VIDEO%.*}_compressed.mkv" # Output filename based on input with '_compressed' suffix and .mkv extension
|
||||
TEMP_DIR="/tmp/ffmpeg_chunks"
|
||||
CHUNK_DURATION=30 # Split video into 30-second chunks for parallel processing
|
||||
NUM_CHUNKS=4 # Limit to 4 chunks
|
||||
|
||||
# Main Process
|
||||
# ========
|
||||
if [[ -z "$INPUT_VIDEO" ]]; then
|
||||
echo "Usage: $0 <input_video>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if GNU Parallel is installed
|
||||
if ! command -v parallel &> /dev/null; then
|
||||
echo "GNU Parallel is required but not installed. Please install it with: apt-get install parallel"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create a temporary directory for storing chunks
|
||||
mkdir -p "$TEMP_DIR"
|
||||
|
||||
# Split the input video into smaller chunks, ensuring proper timestamps and avoiding timestamp issues
|
||||
ffmpeg -fflags +genpts -copyts -i "$INPUT_VIDEO" -c copy -map 0 -segment_time "$CHUNK_DURATION" -reset_timestamps 1 -f segment "$TEMP_DIR/chunk_%03d.mkv"
|
||||
|
||||
# Verify if splitting succeeded
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "Error: Failed to split the video into chunks."
|
||||
rm -rf "$TEMP_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Limit the number of chunks to 4
|
||||
CHUNKS=$(ls "$TEMP_DIR"/chunk_*.mkv | head -n "$NUM_CHUNKS")
|
||||
|
||||
# Compress each chunk in parallel using GNU Parallel
|
||||
echo "$CHUNKS" | parallel -j "$NUM_CHUNKS" ffmpeg -hwaccel cuda -i {} -c:v hevc_nvenc -preset p1 -rc constqp -qp 20 -b:v 5M -maxrate 10M -bufsize 20M -c:a copy {.}_compressed.mkv
|
||||
|
||||
# Verify if compression succeeded
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "Error: Compression failed for one or more chunks."
|
||||
rm -rf "$TEMP_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Concatenate the compressed chunks into the final output file
|
||||
ls "$TEMP_DIR"/*_compressed.mkv | sort | xargs -I {} echo "file '{}'" > "$TEMP_DIR/file_list.txt"
|
||||
ffmpeg -f concat -safe 0 -i "$TEMP_DIR/file_list.txt" -c copy "$OUTPUT_VIDEO"
|
||||
|
||||
# Verify if concatenation succeeded
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "Error: Failed to concatenate the compressed chunks."
|
||||
rm -rf "$TEMP_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Clean up temporary files
|
||||
rm -rf "$TEMP_DIR"
|
||||
|
||||
# Output status
|
||||
if [[ -f "$OUTPUT_VIDEO" ]]; then
|
||||
echo "Compression complete. Output file: $OUTPUT_VIDEO"
|
||||
else
|
||||
echo "Compression failed. Output file was not created."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Display file sizes
|
||||
ls -lh "$INPUT_VIDEO" "$OUTPUT_VIDEO" | awk '{print $9, $5}'
|
||||
|
||||
201
gitea_push.sh
Executable file
201
gitea_push.sh
Executable file
@ -0,0 +1,201 @@
|
||||
#!/bin/zsh
|
||||
# Script Version: 1.3
|
||||
# Description: Pushes the current folder (e.g. /etc) to a nested Gitea repo using provided nesting arguments. Auto-creates the remote repo via Gitea API if missing.
|
||||
|
||||
# Set variables
|
||||
# ========
|
||||
|
||||
# Try to extract GITEA_API_TOKEN from ~/.netrc if present
|
||||
if [ -z "$GITEA_API_TOKEN" ] && grep -q '^GITEA_API_TOKEN=' ~/.netrc 2>/dev/null; then
|
||||
GITEA_API_TOKEN=$(grep '^GITEA_API_TOKEN=' ~/.netrc | head -n1 | cut -d= -f2 | xargs)
|
||||
export GITEA_API_TOKEN
|
||||
fi
|
||||
GITEA_USER="oib"
|
||||
GITEA_URL="https://gitea.bubuit.net"
|
||||
GITEA_API_URL="$GITEA_URL/api/v1"
|
||||
PRIVATE=false
|
||||
DEBUG=false
|
||||
COMMIT_MESSAGE="Update $(date +%F_%T)"
|
||||
|
||||
# Logging function
|
||||
# ========
|
||||
log() {
|
||||
local level="$1"; shift
|
||||
if [ "$level" = "DEBUG" ] && [ "$DEBUG" != true ]; then return; fi
|
||||
local color_reset="$(tput sgr0)"
|
||||
local color=""
|
||||
case "$level" in
|
||||
INFO) color="$(tput setaf 2)" ;; # green
|
||||
WARNING) color="$(tput setaf 3)" ;; # yellow
|
||||
ERROR) color="$(tput setaf 1)" ;; # red
|
||||
DEBUG) color="$(tput setaf 4)" ;; # blue
|
||||
esac
|
||||
echo "${color}[$level] $*${color_reset}"
|
||||
}
|
||||
|
||||
# Functions
|
||||
# ========
|
||||
create_repo() {
|
||||
log INFO "Repository does not exist. Creating via API: $REMOTE_PATH"
|
||||
log DEBUG "POST $GITEA_API_URL/user/repos with name=$REMOTE_PATH and private=$PRIVATE"
|
||||
RESPONSE=$(curl -s -X POST \
|
||||
-H "Authorization: token $GITEA_API_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"name\": \"$FOLDER_NAME\", \"private\": $PRIVATE}" \
|
||||
"$GITEA_API_URL/user/repos")
|
||||
|
||||
if echo "$RESPONSE" | grep -q '"clone_url"'; then
|
||||
log INFO "Remote repository created successfully."
|
||||
else
|
||||
log ERROR "Failed to create remote repository: $RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
prepare_commit() {
|
||||
git add .
|
||||
if ! git rev-parse --verify HEAD >/dev/null 2>&1; then
|
||||
log INFO "Creating initial commit"
|
||||
git commit -m "$COMMIT_MESSAGE"
|
||||
else
|
||||
log INFO "Committing changes"
|
||||
git commit -m "$COMMIT_MESSAGE" || log INFO "Nothing to commit"
|
||||
fi
|
||||
}
|
||||
|
||||
setup_remote() {
|
||||
if git remote | grep -q '^origin$'; then
|
||||
log INFO "Updating remote origin URL"
|
||||
git remote set-url origin "$GIT_REMOTE"
|
||||
else
|
||||
log INFO "Adding remote origin"
|
||||
git remote add origin "$GIT_REMOTE"
|
||||
fi
|
||||
}
|
||||
|
||||
push_changes() {
|
||||
log INFO "Pushing to $GIT_REMOTE"
|
||||
git push -u origin main
|
||||
}
|
||||
|
||||
# Show help if no arguments are given
|
||||
# ========
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "GITEA_API_TOKEN=<your token>"
|
||||
echo "Usage: $0 [--private] [--debug] [--message \"your commit message\"] <host_group>"
|
||||
echo "Example: $0 server"
|
||||
echo " $0 --private workstation"
|
||||
echo " $0 --debug server"
|
||||
echo " $0 --message \"minor update\" server"
|
||||
echo
|
||||
echo "Note: You must cd into the target folder before running this script."
|
||||
echo "For example:"
|
||||
echo " cd /etc && $0 server"
|
||||
echo
|
||||
echo "Authentication:"
|
||||
echo " Git uses ~/.netrc for authentication. You can create it like this:"
|
||||
echo " echo \"machine \$(echo \"$GITEA_URL\" | sed 's|https\\?://||') login $GITEA_USER password \"<your Git token or app password>\"\" > ~/.netrc"
|
||||
echo " chmod 600 ~/.netrc"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Parse arguments
|
||||
# ========
|
||||
POSITIONAL_ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--private)
|
||||
PRIVATE=true
|
||||
shift
|
||||
;;
|
||||
--debug)
|
||||
DEBUG=true
|
||||
shift
|
||||
;;
|
||||
--message)
|
||||
COMMIT_MESSAGE="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
POSITIONAL_ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
set -- "${POSITIONAL_ARGS[@]}"
|
||||
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "Usage: $0 [--private] [--debug] [--message \"your commit message\"] <host_group>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
HOST_GROUP=$(echo "$1" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9-')
|
||||
HOST_NAME=$(hostname -s | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9-')
|
||||
HOST_NAME=$(hostname -s | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9-')
|
||||
FOLDER_NAME="${HOST_NAME}-$(basename "$PWD")"
|
||||
REPO_PATH="$PWD"
|
||||
REMOTE_PATH="$FOLDER_NAME"
|
||||
GIT_REMOTE="https://gitea.bubuit.net/$GITEA_USER/$FOLDER_NAME.git"
|
||||
|
||||
# Git authentication hint
|
||||
# export GIT_ASKPASS=true # disabled: does not affect authentication without handler
|
||||
log DEBUG "Ensure ~/.netrc has:
|
||||
machine gitea.bubuit.net login $GITEA_USER password <personal access token>"
|
||||
|
||||
# Check if GITEA_API_TOKEN is set
|
||||
if [ -z "$GITEA_API_TOKEN" ]; then
|
||||
log WARNING "GITEA_API_TOKEN is not set. Skipping API repo creation."
|
||||
else
|
||||
# Check if remote repo exists
|
||||
HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||
-H "Authorization: token $GITEA_API_TOKEN" \
|
||||
"$GITEA_API_URL/repos/$GITEA_USER/$FOLDER_NAME")
|
||||
|
||||
if [ "$HTTP_STATUS" -ne 200 ]; then
|
||||
create_repo
|
||||
else
|
||||
log INFO "Remote repository already exists."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Main Process
|
||||
# ========
|
||||
|
||||
# Safety check against pushing from / or $HOME
|
||||
if [[ "$PWD" == "$HOME" || "$PWD" == "/" ]]; then
|
||||
log ERROR "Refusing to run inside \$PWD=$PWD"
|
||||
exit 1
|
||||
fi
|
||||
log INFO "Pushing $REPO_PATH to $GIT_REMOTE"
|
||||
cd "$REPO_PATH" || { log ERROR "Directory $REPO_PATH not found"; exit 1; }
|
||||
|
||||
# Initialize git if needed
|
||||
# Branch is fixed to 'main' for simplicity and consistency
|
||||
if [ ! -d .git ]; then
|
||||
log INFO "Initializing Git repo"
|
||||
git init
|
||||
git config init.defaultBranch main
|
||||
git checkout -b main
|
||||
else
|
||||
log DEBUG ".git directory already present"
|
||||
fi
|
||||
|
||||
# Ensure at least one commit exists
|
||||
prepare_commit
|
||||
|
||||
|
||||
# Set or update remote
|
||||
if [ "$HTTP_STATUS" -eq 200 ]; then
|
||||
setup_remote
|
||||
else
|
||||
log WARNING "Skipping remote setup – repository does not exist."
|
||||
fi
|
||||
|
||||
# Push to remote
|
||||
if [ "$HTTP_STATUS" -eq 200 ]; then
|
||||
push_changes
|
||||
else
|
||||
log WARNING "Skipping push – repository does not exist."
|
||||
fi
|
||||
|
||||
46
gitea_push_debug.sh
Executable file
46
gitea_push_debug.sh
Executable file
@ -0,0 +1,46 @@
|
||||
#!/bin/zsh
|
||||
# Script Version: 1.0 Debug
|
||||
# Description: Extended debug version of Gitea push script for diagnosing issues.
|
||||
|
||||
# 1) Basic variables
|
||||
GITEA_USER="oib"
|
||||
GITEA_URL="https://gitea.bubuit.net"
|
||||
GITEA_API_URL="$GITEA_URL/api/v1"
|
||||
|
||||
# 2) Debug function
|
||||
log_debug() {
|
||||
echo "[DEBUG] $@"
|
||||
}
|
||||
|
||||
log_debug "== Starting gitea_push_debug.sh =="
|
||||
|
||||
# 3) Show environment
|
||||
log_debug "Home Dir: $HOME"
|
||||
log_debug "PWD: $PWD"
|
||||
log_debug "User: $USER"
|
||||
|
||||
# 4) Check GITEA_API_TOKEN from environment
|
||||
if [ -z "$GITEA_API_TOKEN" ]; then
|
||||
log_debug "GITEA_API_TOKEN is not set in environment"
|
||||
else
|
||||
log_debug "GITEA_API_TOKEN is present, length: ${#GITEA_API_TOKEN}"
|
||||
fi
|
||||
|
||||
# 5) Attempt to read from ~/.gitea_token
|
||||
if [ -f "$HOME/.gitea_token" ]; then
|
||||
TOKEN_FILE_CONTENT=$(cat "$HOME/.gitea_token")
|
||||
log_debug "~/.gitea_token found, length: ${#TOKEN_FILE_CONTENT}"
|
||||
else
|
||||
log_debug "~/.gitea_token not found"
|
||||
fi
|
||||
|
||||
# 6) Try an API request to /user with the token from environment
|
||||
if [ -n "$GITEA_API_TOKEN" ]; then
|
||||
USER_RESPONSE=$(curl -s -H "Authorization: token $GITEA_API_TOKEN" "$GITEA_API_URL/user")
|
||||
log_debug "Response from /user: $USER_RESPONSE"
|
||||
else
|
||||
log_debug "Skipping /user request; no valid GITEA_API_TOKEN in environment."
|
||||
fi
|
||||
|
||||
log_debug "== End gitea_push_debug.sh =="
|
||||
|
||||
63
normalize.sh
Executable file
63
normalize.sh
Executable file
@ -0,0 +1,63 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Check for ffmpeg dependency
|
||||
if ! command -v ffmpeg &> /dev/null; then
|
||||
echo "ffmpeg could not be found. Please install ffmpeg to use this script."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Number of concurrent ffmpeg processes
|
||||
N=${1:-8}
|
||||
|
||||
# Semaphore file for limiting concurrent processes
|
||||
semaphore=/tmp/ffmpeg.lock
|
||||
|
||||
# Opening the semaphore file descriptor on 9
|
||||
exec 9>"$semaphore"
|
||||
|
||||
# Function to run an ffmpeg command with semaphore locking
|
||||
run_ffmpeg() {
|
||||
# Wait for a semaphore slot to become available
|
||||
flock -x 9
|
||||
|
||||
# Execute the ffmpeg command
|
||||
ffmpeg -y -i "$1" -af "loudnorm=I=-16:LRA=11:measured_I=-20:measured_LRA=16,volume=0.8" "$2"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "An error occurred with ffmpeg processing $1"
|
||||
# Release the semaphore slot on error as well
|
||||
flock -u 9
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Release the semaphore slot
|
||||
flock -u 9
|
||||
}
|
||||
|
||||
# Create the semaphore file if it does not exist
|
||||
touch "$semaphore"
|
||||
if [ ! -f "$semaphore" ]; then
|
||||
echo "Failed to create semaphore file."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Processing each .opus file
|
||||
find . -maxdepth 1 -type f -name '*.m4a' | while read -r file; do
|
||||
wav_file="${file/%.m4a/.wav}"
|
||||
if [ ! -f "$wav_file" ]; then
|
||||
echo "Processing $file..."
|
||||
run_ffmpeg "$file" "$wav_file" &
|
||||
|
||||
# Ensure N parallel ffmpeg processes
|
||||
while [ $(jobs -p | wc -l) -ge "$N" ]; do
|
||||
wait -n
|
||||
done
|
||||
fi
|
||||
done
|
||||
|
||||
# Wait for all background jobs to finish
|
||||
wait
|
||||
|
||||
# Close the semaphore file descriptor and remove the file
|
||||
exec 9>&-
|
||||
rm -f "$semaphore"
|
||||
|
||||
189
scraper.sh
Executable file
189
scraper.sh
Executable file
@ -0,0 +1,189 @@
|
||||
#!/bin/bash
|
||||
# Script Version: 01.8
|
||||
# Description: Scrapes and extracts page text from MediaWiki pages, cleans image artifacts, and deletes empty results
|
||||
|
||||
# Constants
|
||||
DOWNLOAD_TIMEOUT=10
|
||||
TEXT_FILE_SUFFIX=".txt"
|
||||
LINK_FILE_SUFFIX=".txt"
|
||||
|
||||
# Function to convert relative URLs to absolute URLs
|
||||
resolve_url() {
|
||||
local base_url=$1
|
||||
local relative_url=$2
|
||||
|
||||
if [[ "$relative_url" =~ ^https?:// ]]; then
|
||||
echo "$relative_url"
|
||||
elif [[ "$relative_url" =~ ^/ ]]; then
|
||||
echo "${base_url}${relative_url}"
|
||||
else
|
||||
echo "${base_url}/${relative_url}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to check if URL should be skipped
|
||||
should_skip_url() {
|
||||
local url=$1
|
||||
case "$url" in
|
||||
*"load.php"*|*"IE9fixes.css"*|*"favicon.ico"*|*"opensearch_desc.php"*|*"api.php?action="*|*"Special:RecentChanges"*|*"Special:UserLogin"*|*"Special:RequestAccount"*|*"Dioxipedia:Privacy_policy"*|*"javascript:print();"*|*"mediawiki.org"*)
|
||||
return 0 ;; # true, should skip
|
||||
*)
|
||||
return 1 ;; # false, don't skip
|
||||
esac
|
||||
}
|
||||
|
||||
# Function to download content into a variable with timeout and error checking
|
||||
download_content() {
|
||||
local url=$1
|
||||
local exclude_file=$2
|
||||
|
||||
if should_skip_url "$url"; then
|
||||
echo "Skipping known irrelevant URL: $url"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ -f "$exclude_file" ] && grep -Fx "$url" "$exclude_file" > /dev/null; then
|
||||
echo "Skipping excluded URL: $url"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Downloading: $url"
|
||||
SITECACHE=$(wget -T "$DOWNLOAD_TIMEOUT" -q -O - "$url" 2>/dev/null)
|
||||
if [ $? -ne 0 ] || [ -z "$SITECACHE" ]; then
|
||||
echo -e "\033[31m[ ERROR ]:\033[0m Failed to download $url" >&2
|
||||
echo "$url" >> "$exclude_file"
|
||||
return 1
|
||||
fi
|
||||
if ! echo "$SITECACHE" | grep -q "<html"; then
|
||||
echo "Skipping: $url (not HTML)"
|
||||
echo "$url" >> "$exclude_file"
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
echo "Successfully downloaded: $url"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Improved extraction function using pup and lynx
|
||||
extract_text() {
|
||||
local output_file=$1
|
||||
local url=$2
|
||||
local exclude_file=$3
|
||||
|
||||
echo "Extracting text from SITECACHE to $output_file"
|
||||
|
||||
EXTRACTED=$(echo "$SITECACHE" | pup '#mw-content-text' 2>/dev/null)
|
||||
|
||||
if [ -z "$(echo "$EXTRACTED" | sed '/^\s*$/d')" ]; then
|
||||
echo "INFO: Content empty with #mw-content-text, trying #bodyContent"
|
||||
EXTRACTED=$(echo "$SITECACHE" | pup '#bodyContent' 2>/dev/null)
|
||||
fi
|
||||
|
||||
if [ -z "$(echo "$EXTRACTED" | sed '/^\s*$/d')" ]; then
|
||||
echo "WARNING: Still no content after both selectors."
|
||||
echo "$url" >> "$exclude_file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "$EXTRACTED" | lynx -stdin -dump -nolist > "$output_file"
|
||||
|
||||
if [ ! -s "$output_file" ]; then
|
||||
echo "WARNING: No text extracted from $url after lynx"
|
||||
echo "$url" >> "$exclude_file"
|
||||
rm -f "$output_file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Remove lines containing image artifacts like [something.jpg] or [something.png]
|
||||
sed -i '/\[.*\(jpg\|jpeg\|png\).*]/Id' "$output_file"
|
||||
|
||||
# Delete if file is smaller than 100 bytes
|
||||
if [ $(stat -c%s "$output_file") -lt 100 ]; then
|
||||
echo "INFO: Deleted $output_file (under 100 bytes)"
|
||||
rm -f "$output_file"
|
||||
echo "$url" >> "$exclude_file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Successfully extracted text to $output_file"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Function to extract page title
|
||||
extract_title() {
|
||||
echo "$SITECACHE" | grep -oP '(?<=<title>).*(?=</title>)' | head -n 1 | sed 's/ - dioxipedia$//' | sed 's/[^a-zA-Z0-9-]/_/g' | sed 's/__*/_/g' | sed 's/^_//;s/_$//'
|
||||
}
|
||||
|
||||
# Function to extract links
|
||||
extract_links() {
|
||||
local output_file=$1
|
||||
|
||||
echo "$SITECACHE" | grep -oP '(?<=href=")[^"]+' | grep -v 'translate\.goog' > "$output_file"
|
||||
if [ $? -ne 0 ] || [ ! -s "$output_file" ]; then
|
||||
echo "WARNING: No links extracted"
|
||||
rm -f "$output_file"
|
||||
return 1
|
||||
fi
|
||||
echo "Successfully extracted links to $output_file"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Main script logic
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "Usage: $0 <URL>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
INITIAL_URL=$1
|
||||
DOMAIN=$(echo "$INITIAL_URL" | awk -F[/:] '{print $4}')
|
||||
BASE_URL="https://$DOMAIN"
|
||||
ALL_PAGES_URL="$BASE_URL/index.php?title=Special:AllPages"
|
||||
LINKSFILE="$DOMAIN/links$LINK_FILE_SUFFIX"
|
||||
EXCLUDE_FILE="$DOMAIN/exclude.txt"
|
||||
CONTENT_DIR="$DOMAIN/content"
|
||||
|
||||
mkdir -p "$DOMAIN"
|
||||
mkdir -p "$CONTENT_DIR"
|
||||
|
||||
# Step 1: Collect links
|
||||
if ! download_content "$ALL_PAGES_URL" "$EXCLUDE_FILE"; then
|
||||
echo "Failed to download $ALL_PAGES_URL"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! extract_links "$LINKSFILE"; then
|
||||
echo "Failed to extract links"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Step 2: Process links
|
||||
PROCESSED_URLS=()
|
||||
while IFS= read -r link; do
|
||||
URL=$(resolve_url "$BASE_URL" "$link")
|
||||
|
||||
if [[ " ${PROCESSED_URLS[*]} " =~ " $URL " ]]; then
|
||||
echo "Skipping processed URL: $URL"
|
||||
continue
|
||||
fi
|
||||
|
||||
if ! download_content "$URL" "$EXCLUDE_FILE"; then
|
||||
PROCESSED_URLS+=("$URL")
|
||||
continue
|
||||
fi
|
||||
|
||||
PAGENAME=$(extract_title)
|
||||
[ -z "$PAGENAME" ] && PAGENAME="page"
|
||||
|
||||
TEXTFILE="$CONTENT_DIR/$PAGENAME$TEXT_FILE_SUFFIX"
|
||||
|
||||
if ! extract_text "$TEXTFILE" "$URL" "$EXCLUDE_FILE"; then
|
||||
PROCESSED_URLS+=("$URL")
|
||||
continue
|
||||
fi
|
||||
|
||||
PROCESSED_URLS+=("$URL")
|
||||
done < "$LINKSFILE"
|
||||
|
||||
echo "Processing complete."
|
||||
exit 0
|
||||
|
||||
12
set_nvidia_fan.sh
Executable file
12
set_nvidia_fan.sh
Executable file
@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
# Script Version: 03
|
||||
# Description: Set NVIDIA GPU fan speed on startup.
|
||||
|
||||
# Define the display and X authority paths
|
||||
export DISPLAY=:1
|
||||
export XAUTHORITY=/run/user/1000/gdm/Xauthority
|
||||
|
||||
# Run nvidia-settings commands as root
|
||||
sudo /usr/bin/nvidia-settings -c :1 -a "[gpu:0]/GPUFanControlState=1"
|
||||
sudo /usr/bin/nvidia-settings -c :1 -a "[fan:0]/GPUTargetFanSpeed=10"
|
||||
|
||||
47
soa_monitor.sh
Executable file
47
soa_monitor.sh
Executable file
@ -0,0 +1,47 @@
|
||||
#!/bin/bash
|
||||
# Script Version: 07
|
||||
# Description: Monitor SOA for dynproxy.net and play a sound when it changes to ns1.dynproxy.net.
|
||||
|
||||
# Set variables
|
||||
# ========
|
||||
DOMAIN="dynproxy.net"
|
||||
EXPECTED_NS="ns1.dynproxy.net."
|
||||
SOUND_CMD="paplay /usr/share/sounds/freedesktop/stereo/alarm-clock-elapsed.oga"
|
||||
CHECK_INTERVAL=60 # Interval in seconds
|
||||
|
||||
# Functions
|
||||
# ========
|
||||
get_soa() {
|
||||
dig "$DOMAIN" SOA +short | awk '{print $1}'
|
||||
}
|
||||
|
||||
play_sound() {
|
||||
$SOUND_CMD
|
||||
}
|
||||
|
||||
# Main Process
|
||||
# ========
|
||||
echo "Monitoring SOA for $DOMAIN. Expected NS: $EXPECTED_NS"
|
||||
LAST_SOA=""
|
||||
|
||||
while true; do
|
||||
CURRENT_SOA=$(get_soa)
|
||||
|
||||
if [[ -z "$CURRENT_SOA" ]]; then
|
||||
echo "Error fetching SOA record. Network issue or domain unreachable."
|
||||
sleep $CHECK_INTERVAL
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$CURRENT_SOA" != "$LAST_SOA" ]]; then
|
||||
echo "SOA changed! New SOA: $CURRENT_SOA"
|
||||
LAST_SOA="$CURRENT_SOA"
|
||||
|
||||
if [[ "$CURRENT_SOA" == "$EXPECTED_NS" ]]; then
|
||||
echo "SOA matches expected NS. Playing sound..."
|
||||
play_sound
|
||||
fi
|
||||
fi
|
||||
sleep $CHECK_INTERVAL
|
||||
done
|
||||
|
||||
41
test_ns_zones.sh
Executable file
41
test_ns_zones.sh
Executable file
@ -0,0 +1,41 @@
|
||||
#!/bin/zsh
|
||||
# Version 01.0
|
||||
# Script to test DNS zone propagation across ns1, ns2, and ns3
|
||||
# Script Name: test_ns_zones.sh
|
||||
|
||||
# Variables
|
||||
NS1="23.88.113.138"
|
||||
NS2="116.202.112.180"
|
||||
NS3="95.216.198.140"
|
||||
|
||||
# Check if a domain name argument is provided
|
||||
if [ -z "$1" ]; then
|
||||
echo "Usage: $0 <domain.tld>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DOMAIN=$1
|
||||
|
||||
# Function to test a DNS query
|
||||
function test_ns {
|
||||
local NS=$1
|
||||
echo "
|
||||
=== Testing $DOMAIN on $NS ==="
|
||||
dig @$NS $DOMAIN SOA +short
|
||||
echo ""
|
||||
echo "MX Record:"
|
||||
dig @$NS $DOMAIN MX +short
|
||||
echo ""
|
||||
echo "A Record for mail.$DOMAIN:"
|
||||
dig @$NS mail.$DOMAIN A +short
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Test each nameserver
|
||||
test_ns $NS1
|
||||
test_ns $NS2
|
||||
test_ns $NS3
|
||||
|
||||
# Success message
|
||||
echo "DNS zone test completed for $DOMAIN"
|
||||
|
||||
43
wrap_embeddings.sh
Executable file
43
wrap_embeddings.sh
Executable file
@ -0,0 +1,43 @@
|
||||
#!/bin/bash
|
||||
# Script Version: 0.3
|
||||
# Description: Convert each .txt in content/ to .json with embedding in json/
|
||||
|
||||
# Set variables
|
||||
CONTENT_DIR="./content"
|
||||
JSON_DIR="./json"
|
||||
EMBEDDING_MODEL="sentence-transformers/all-MiniLM-L6-v2"
|
||||
|
||||
# Check dependencies
|
||||
if ! python3 -c "import sentence_transformers" 2>/dev/null; then
|
||||
echo "[ERROR] ❌ sentence-transformers not installed. Run: pip3 install sentence-transformers"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check input files
|
||||
mkdir -p "$JSON_DIR"
|
||||
if [ ! -d "$CONTENT_DIR" ] || ! ls "$CONTENT_DIR"/*.txt >/dev/null 2>&1; then
|
||||
echo "[ERROR] ❌ No .txt files found in $CONTENT_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Generate embeddings
|
||||
python3 -c "
|
||||
import sys, json, os
|
||||
from sentence_transformers import SentenceTransformer
|
||||
content_dir, json_dir = sys.argv[1], sys.argv[2]
|
||||
model = SentenceTransformer('${EMBEDDING_MODEL}')
|
||||
for txt_file in os.listdir(content_dir):
|
||||
if txt_file.endswith('.txt'):
|
||||
base_name = txt_file[:-4]
|
||||
try:
|
||||
with open(os.path.join(content_dir, txt_file), 'r', encoding='utf-8') as f:
|
||||
text = f.read()
|
||||
embedding = model.encode([text])[0].tolist()
|
||||
with open(os.path.join(json_dir, f'{base_name}.json'), 'w') as f:
|
||||
json.dump({'id': base_name, 'text': text, 'embedding': embedding}, f)
|
||||
print(f'[DEBUG] ✅ Saved: {json_dir}/{base_name}.json')
|
||||
except Exception as e:
|
||||
print(f'[ERROR] ❌ Failed: {txt_file} - {str(e)}', file=sys.stderr)
|
||||
" "$CONTENT_DIR" "$JSON_DIR" 2>&1 | while read -r line; do echo "$line"; done
|
||||
|
||||
echo "✅ All .txt files converted to JSON with embeddings in $JSON_DIR"
|
||||
Reference in New Issue
Block a user