diff --git a/conf/garage.toml b/conf/garage.toml index 2cae1f9..beb3a68 100644 --- a/conf/garage.toml +++ b/conf/garage.toml @@ -1,5 +1,5 @@ -metadata_dir = "/opt/yunohost/__APP_NAME__/metadata" -data_dir = "__DATA_DIR__" +metadata_dir = "/opt/yunohost/__APP__/metadata" +data_dir = "__DATADIR__" block_size = 1048576 block_manager_background_tranquility = 2 @@ -10,14 +10,13 @@ compression_level = 1 # openssl rand -hex 32 rpc_secret = "__RPC_SECRET__" -# LE PORT DOIT ÊTRE LE MÊME SUR TOUS LES NOEUDS (port interne) rpc_bind_addr = "[::]:__PORT__" # Le port peut être différent (eg NAT) mais doit rediriger sur le # port de rpc_bind_addr -rpc_public_addr = "__IP__:__PORT__" +#rpc_public_addr = "__IP__:__PORT__" bootstrap_peers = [ - __SERVER_AMI__", + __BOOTSTRAP_PEERS_VAR__ ] [s3_api] @@ -27,7 +26,7 @@ api_bind_addr = "[::]:__PORT_API__" s3_region = "garage" [s3_web] -bind_addr = "[::]:__PORT_WEB" +bind_addr = "[::]:__PORT_WEB__" root_domain = ".web.garage.localhost" index = "index.html" diff --git a/conf/mount_disk.sh b/conf/mount_disk.sh index 5ddf0e8..59a1b60 100755 --- a/conf/mount_disk.sh +++ b/conf/mount_disk.sh @@ -1,16 +1,19 @@ #!/bin/bash -datadir=$1 -format=$2 -i=0 -while $(fdisk -l /dev/nbd$i 1&>2 /dev/null) -do - i=$(( i + 1 )) -done -echo $i -modprobe nbd max_part=$(( i + 1 )) -qemu-nbd --connect /dev/nbd$i $datadir/garage.qcow2 -if [[ "$format" = "true" ]] +if [ "$VIRTUALISATION" = "true" ] then - mkfs.ext4 /dev/nbd$i > /dev/null + datadir=__DATADIR__ + format=$1 + i=0 + while $(fdisk -l /dev/nbd$i 1&>2 /dev/null) + do + i=$(( i + 1 )) + done + echo $i + modprobe nbd max_part=$(( i + 1 )) + qemu-nbd --connect /dev/nbd$i $datadir/garage.qcow2 + if [[ "$format" = "true" ]] + then + mkfs.ext4 /dev/nbd$i > /dev/null + fi + mount /dev/nbd$i $datadir/data/ fi -mount /dev/nbd$i $datadir/data/ diff --git a/conf/systemd.service b/conf/systemd.service index b30bee9..0702044 100644 --- a/conf/systemd.service +++ b/conf/systemd.service @@ -5,12 +5,10 @@ Wants=network-online.target [Service] User=__APP__ -Environment='RUST_LOG=garage=info' 'RUST_BACKTRACE=1' -ExecStartPre=+__FINALPATH__/mount_disk.sh __FINALPATH__ __NBD_INDEX__ +Environment='RUST_LOG=garage=info' 'VIRTUALISATION=__VIRTUALISATION__' 'RUST_BACKTRACE=1' +ExecStartPre=+__FINALPATH__/mount_disk.sh ExecStart=__FINALPATH__/garage -c __FINALPATH__/garage.toml server -StateDirectory=garage -DynamicUser=true -ProtectHome=true +WorkingDirectory=__FINALPATH__/ NoNewPrivileges=true StandardOutput=append:/var/log/__APP__/__APP__.log StandardError=inherit @@ -21,7 +19,7 @@ StandardError=inherit # Details for these options: https://www.freedesktop.org/software/systemd/man/systemd.exec.html PrivateTmp=yes PrivateDevices=yes -RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +# RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 RestrictNamespaces=yes RestrictRealtime=yes DevicePolicy=closed diff --git a/conf/umount_disk.sh b/conf/umount_disk.sh index c6cc5ed..1e11d56 100755 --- a/conf/umount_disk.sh +++ b/conf/umount_disk.sh @@ -1,4 +1,7 @@ #!/bin/bash -nbd=$1 -umount /dev/nbd$nbd -qemu-nbd --disconnect /dev/nbd$nbd +if [ "$VIRTUALISATION" = "true" ] +then + nbd=__NBD_INDEX__ + umount /dev/nbd$nbd + qemu-nbd --disconnect /dev/nbd$nbd +fi diff --git a/manifest.json b/manifest.json index 7902091..4af1b22 100644 --- a/manifest.json +++ b/manifest.json @@ -6,7 +6,7 @@ "en": "S3 storage", "fr": "stockage S3" }, - "version": "0.7.2.1~ynh1", + "version": "0.7.3~ynh1", "url": "https://garagehq.deuxfleurs.fr/", "upstream": { "license": "free", @@ -32,7 +32,7 @@ "type": "domain" }, { - "name":"rpc-secret", + "name":"rpc_secret", "type":"string", "ask": { "en": "UUID of the network (rpc-secret) ", @@ -42,7 +42,7 @@ "example": "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec" }, { - "name": "bootstrap-peers", + "name": "bootstrap_peers", "type": "string", "example": "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec@127.0.0.1:3901", "optional": true, @@ -53,7 +53,7 @@ }, { "name": "weight", - "type": "integer", + "type": "string", "ask": { "en": "number of G to allow", "fr": "nombre de G à allouer" diff --git a/scripts/_common.sh b/scripts/_common.sh index 0642659..5130faa 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -3,14 +3,14 @@ #================================================= # COMMON VARIABLES #================================================= -pkg_dependencies="qemu-utils" +pkg_dependencies_virtualisation="qemu-utils" #================================================= # PERSONAL HELPERS #================================================= -GARAGE_VERSION="0.7.2.1" +GARAGE_VERSION="0.7.3" get_ip() { curl ip.me @@ -37,9 +37,8 @@ install_garage () { ynh_die --message="Unsupported architecture \"$architecture\"" ;; esac - wget https://garagehq.deuxfleurs.fr/_releases/v$GARAGE_VERSION/$arch-unknown-linux-musl/garage -O "$app" 2>&1 >/dev/null - chmod +x "$app" - fi + wget https://garagehq.deuxfleurs.fr/_releases/v$GARAGE_VERSION/$arch-unknown-linux-musl/garage -O garage 2>&1 >/dev/null + chmod +x garage } init_garage() { @@ -54,145 +53,65 @@ apply_layout "$garage_command" } apply_layout() { - $garage_command=$1 - $garage_command -c garage.toml layout show 2>/dev/null | grep "--version" - if [ $? -eq 0 ] + garage_command=$1 + + if [ $($garage_command -c garage.toml layout show 2>/dev/null | grep -- --version) ] then layout_version=$($garage_command layout show 2>/dev/null | grep -Po -- "(?<=--version).*" | head -1 | xargs) $garage_command layout apply --version $layout_version else - return 1 + return 0 fi } #================================================= # EXPERIMENTAL HELPERS #================================================= -# Send an email to inform the administrator -# -# usage: ynh_send_readme_to_admin --app_message=app_message [--recipients=recipients] [--type=type] -# | arg: -m --app_message= - The file with the content to send to the administrator. -# | arg: -r, --recipients= - The recipients of this email. Use spaces to separate multiples recipients. - default: root -# example: "root admin@domain" -# If you give the name of a YunoHost user, ynh_send_readme_to_admin will find its email adress for you -# example: "root admin@domain user1 user2" -# | arg: -t, --type= - Type of mail, could be 'backup', 'change_url', 'install', 'remove', 'restore', 'upgrade' -# -# Requires YunoHost version 4.1.0 or higher. ynh_send_readme_to_admin() { - # Declare an array to define the options of this helper. - declare -Ar args_array=( [m]=app_message= [r]=recipients= [t]=type= ) - local app_message - local recipients - local type - # Manage arguments with getopts + local app_message="${1:-...No specific information...}" + local recipients="${2:-root}" - ynh_handle_getopts_args "$@" - app_message="${app_message:-}" - recipients="${recipients:-root}" - type="${type:-install}" + # Retrieve the email of users + find_mails () { + local list_mails="$1" + local mail + local recipients=" " + # Read each mail in argument + for mail in $list_mails + do + # Keep root or a real email address as it is + if [ "$mail" = "root" ] || echo "$mail" | grep --quiet "@" + then + recipients="$recipients $mail" + else + # But replace an user name without a domain after by its email + if mail=$(ynh_user_get_info "$mail" "mail" 2> /dev/null) + then + recipients="$recipients $mail" + fi + fi + done + echo "$recipients" + } + recipients=$(find_mails "$recipients") - # Get the value of admin_mail_html - admin_mail_html=$(ynh_app_setting_get $app admin_mail_html) - admin_mail_html="${admin_mail_html:-0}" + local mail_subject="☁️🆈🅽🅷☁️: \`$app\` was just installed!" - # Retrieve the email of users - find_mails () { - local list_mails="$1" - local mail - local recipients=" " - # Read each mail in argument - for mail in $list_mails - do - # Keep root or a real email address as it is - if [ "$mail" = "root" ] || echo "$mail" | grep --quiet "@" - then - recipients="$recipients $mail" - else - # But replace an user name without a domain after by its email - if mail=$(ynh_user_get_info "$mail" "mail" 2> /dev/null) - then - recipients="$recipients $mail" - fi - fi - done - echo "$recipients" - } - recipients=$(find_mails "$recipients") - - # Subject base - local mail_subject="☁️🆈🅽🅷☁️: \`$app\`" - - # Adapt the subject according to the type of mail required. - if [ "$type" = "backup" ]; then - mail_subject="$mail_subject has just been backup." - elif [ "$type" = "change_url" ]; then - mail_subject="$mail_subject has just been moved to a new URL!" - elif [ "$type" = "remove" ]; then - mail_subject="$mail_subject has just been removed!" - elif [ "$type" = "restore" ]; then - mail_subject="$mail_subject has just been restored!" - elif [ "$type" = "upgrade" ]; then - mail_subject="$mail_subject has just been upgraded!" - else # install - mail_subject="$mail_subject has just been installed!" - fi - - ynh_add_config --template="$app_message" --destination="../conf/msg_to_send" - - ynh_delete_file_checksum --file="../conf/msg_to_send" - local mail_message="This is an automated message from your beloved YunoHost server. + local mail_message="This is an automated message from your beloved YunoHost server. Specific information for the application $app. -$(cat "../conf/msg_to_send")" +$app_message" - # Store the message into a file for further modifications. - echo "$mail_message" > mail_to_send + # Define binary to use for mail command + if [ -e /usr/bin/bsd-mailx ] + then + local mail_bin=/usr/bin/bsd-mailx + else + local mail_bin=/usr/bin/mail.mailutils + fi - # If a html email is required. Apply html tags to the message. - if [ "$admin_mail_html" -eq 1 ] - then - # Insert 'br' tags at each ending of lines. - ynh_replace_string "$" "
" mail_to_send - - # Insert starting HTML tags - sed --in-place '1s@^@\n\n\n\n@' mail_to_send - - # Keep tabulations - ynh_replace_string " " "\ \ " mail_to_send - ynh_replace_string "\t" "\ \ " mail_to_send - - # Insert url links tags - ynh_replace_string "__URL_TAG1__\(.*\)__URL_TAG2__\(.*\)__URL_TAG3__" "\1" mail_to_send - - # Insert finishing HTML tags - echo -e "\n\n" >> mail_to_send - - # Otherwise, remove tags to keep a plain text. - else - # Remove URL tags - ynh_replace_string "__URL_TAG[1,3]__" "" mail_to_send - ynh_replace_string "__URL_TAG2__" ": " mail_to_send - fi - - # Define binary to use for mail command - if [ -e /usr/bin/bsd-mailx ] - then - local mail_bin=/usr/bin/bsd-mailx - else - local mail_bin=/usr/bin/mail.mailutils - fi - - if [ "$admin_mail_html" -eq 1 ] - then - content_type="text/html" - else - content_type="text/plain" - fi - - # Send the email to the recipients - cat mail_to_send | $mail_bin -a "Content-Type: $content_type; charset=UTF-8" -s "$mail_subject" "$recipients" + # Send the email to the recipients + echo "$mail_message" | $mail_bin -a "Content-Type: text/plain; charset=UTF-8" -s "$mail_subject" "$recipients" } - #================================================= # FUTURE OFFICIAL HELPERS #================================================= diff --git a/scripts/backup b/scripts/backup index 91261a6..c89a92e 100755 --- a/scripts/backup +++ b/scripts/backup @@ -56,7 +56,7 @@ ynh_backup --src_path="$final_path" # BACKUP THE DATA DIR #================================================= -ynh_backup --src_path="$datadir" --is_big +#ynh_backup --src_path="$datadir/data" --is_big #================================================= # BACKUP THE NGINX CONFIGURATION diff --git a/scripts/install b/scripts/install index 4b69c0a..b88eb6b 100755 --- a/scripts/install +++ b/scripts/install @@ -34,6 +34,7 @@ bootstrap_peers=$YNH_APP_ARG_BOOTSTRAP_PEERS datadir=$YNH_APP_ARG_DATADIR weight=$YNH_APP_ARG_WEIGHT + ### If it's a multi-instance app, meaning it can be installed several times independently ### The id of the app as stated in the manifest is available as $YNH_APP_ID ### The instance number is available as $YNH_APP_INSTANCE_NUMBER (equals "1", "2"...) @@ -73,6 +74,9 @@ fi if [ -n "$bootstrap_peers" ] then echo "$bootstrap_peers" | grep -E '[0-9a-f]{64}@(\b25[0-5]|\b2[0-4][0-9]|\b[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}:3901' || ynh_die --message="friend server id must have id with the following form : 1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec@192.168.1.1:3901" + bootstrap_peers_var="\"$bootstrap_peers\"," +else + bootstrap_peers_var="" fi if [ "$datadir" = "/home/yunohost.app/__APP_NAME__/data" ] @@ -83,6 +87,19 @@ fi # Register (book) web path ynh_webpath_register --app=$app --domain=$domain --path_url="/" +#================================================= +# LOOKING FOR VIRTUALISATION +#================================================= +ynh_script_progression --message="Checking virtualisation availability…" --time --weight=1 + +if [ "$(which modprobe)" = "" ] +then + virtualisation=false +else + virtualisation=true +fi +ynh_app_setting_set --app=$app --key=virtualisation --value=true + #================================================= # STORE SETTINGS FROM MANIFEST #================================================= @@ -108,15 +125,18 @@ ynh_script_progression --message="Finding available ports..." --time --weight=1 # Find an available port -port=$(ynh_find_port --port=3901) -port_api=$(ynh_find_port --port=3900) -port_web=$(ynh_find_port --port=3902) -port_admin=$(ynh_find_port --port=3903) - +port=$(ynh_find_port --port=4000) ynh_app_setting_set --app=$app --key=port --value=$port +port_api=$(ynh_find_port --port=5000) ynh_app_setting_set --app=$app --key=port_api --value=$port_api +port_web=$(ynh_find_port --port=6000) ynh_app_setting_set --app=$app --key=port_web --value=$port_web +port_admin=$(ynh_find_port --port=7000) ynh_app_setting_set --app=$app --key=port_admin --value=$port_admin + +ynh_print_warn --message="port : $port port_api : $port_api port_web : $port_web port_admin : $port_admin" + + # Optional: Expose this port publicly # (N.B.: you only need to do this if the app actually needs to expose the port publicly. # If you do this and the app doesn't actually need you are CREATING SECURITY HOLES IN THE SERVER !) @@ -137,8 +157,10 @@ ynh_script_progression --message="Installing dependencies..." --time --weight=1 ### - Remove the variable "pkg_dependencies" in _common.sh ### - As well as the section "REINSTALL DEPENDENCIES" in the restore script ### - And the section "UPGRADE DEPENDENCIES" in the upgrade script - ynh_install_app_dependencies $pkg_dependencies - +if [ "$virtualisation" = "true" ] +then + ynh_install_app_dependencies $pkg_dependencies_virtualisation +fi #================================================= # CREATE DEDICATED USER #================================================= @@ -156,7 +178,9 @@ ynh_script_progression --message="Setting up source files..." --time --weight=1 ### downloaded from an upstream source, like a git repository. ### `ynh_setup_source` use the file conf/app.src +mkdir -p $final_path ynh_app_setting_set --app=$app --key=final_path --value=$final_path + # Download, check integrity, uncompress and patch the source from app.src pushd $final_path install_garage @@ -212,10 +236,14 @@ mkdir -p $datadir/data #================================================= # create data partition #================================================= -# to be sure to not exceed size limit, i use a virtual disk with a fix size to have a max limit size. -qemu-img create -f qcow2 $datadir/garage_data.qcow2 "$weight"G -nbd_index=$($final_path/mount_disk.sh "$datadir" true) -ynh_app_setting_set --app=$app --key=nbd_index --value=$nbd_index +nbd_index=127 +if [ "$virtualisation" = "true" ] +then + # to be sure to not exceed size limit, i use a virtual disk with a fix size to have a max limit size. + qemu-img create -f qcow2 $datadir/garage_data.qcow2 "$weight"G + nbd_index=$($final_path/mount_disk.sh true) + ynh_app_setting_set --app=$app --key=nbd_index --value=$nbd_index +fi # FIXME: this should be managed by the core in the future # Here, as a packager, you may have to tweak the ownerhsip/permissions @@ -227,7 +255,10 @@ chmod 750 "$datadir" chmod -R o-rwx "$datadir" chown -R $app:$app "$datadir" -$final_path/umount_disk.sh $nbd_index +if [ "$virtualisation" = "true" ] +then + $final_path/umount_disk.sh $nbd_index +fi #================================================= # ADD A CONFIGURATION #================================================= @@ -245,13 +276,16 @@ ynh_script_progression --message="Adding a configuration file..." --time --weigh ### ### Check the documentation of `ynh_add_config` for more info. -ynh_add_config --template="garage.toml" --destination="$final_path/garage.toml" + ynh_add_config --template="mount_disk.sh" --destination="$final_path/mount_disk.sh" + ynh_add_config --template="umount_disk.sh" --destination="$final_path/umount_disk.sh" + ynh_add_config --template="garage.toml" --destination="$final_path/garage.toml" + chmod +x "$final_path/mount_disk.sh" "$final_path/umount_disk.sh" # FIXME: this should be handled by the core in the future # You may need to use chmod 600 instead of 400, # for example if the app is expected to be able to modify its own config -chmod 400 "$final_path/garage.toml" -chown $app:$app "$final_path/garag.toml" +chmod 600 "$final_path/garage.toml" +chown $app:$app "$final_path/garage.toml" ### For more complex cases where you want to replace stuff using regexes, ### you shoud rely on ynh_replace_string (which is basically a wrapper for sed) @@ -311,7 +345,7 @@ ynh_script_progression --message="Integrating service in YunoHost..." --time --w ### - As well as the section "INTEGRATE SERVICE IN YUNOHOST" in the restore script ### - And the section "INTEGRATE SERVICE IN YUNOHOST" in the upgrade script -yunohost service add --needs_exposed_ports $app --description="s3 storage" --log="/var/log/$app/$app.log" +yunohost service add --needs_exposed_ports $port --description="s3 storage" --log="/var/log/$app/$app.log" $app ### Additional options starting with 3.8: ### @@ -346,6 +380,7 @@ ynh_script_progression --message="Starting a systemd service..." --time --weight # Start a systemd service ynh_systemd_action --service_name=$app --action="start" --log_path="/var/log/$app/$app.log" + #================================================= # SETUP SSOWAT #================================================= @@ -368,10 +403,11 @@ ynh_systemd_action --service_name=nginx --action=reload #================================================= ynh_script_progression --message="Configuring garage..." --time --weight=1 +garage_command="$final_path/garage -c $final_path/garage.toml" + node_id=$($garage_command node id -q | cut -d '@' -f1) ynh_app_setting_set --app=$app --key=node_id --value=node_id -garage_command="$garage_path/garage -c $garage_path/garage.toml" init_garage "$garage_command" "$node_id" "$weight" "$domain" diff --git a/scripts/remove b/scripts/remove index 0f79639..5bb099e 100755 --- a/scripts/remove +++ b/scripts/remove @@ -23,10 +23,13 @@ port_api=$(ynh_app_setting_get --app=$app --key=port_api) port_web=$(ynh_app_setting_get --app=$app --key=port_web) datadir=$(ynh_app_setting_get --app=$app --key=datadir) bootstrap_peers=$(ynh_app_setting_get --app=$app --key=bootstrap_peers) -nbd_index=$(ynh_app_setting_get --app=$app --key=nbd_index) node_id=$(ynh_app_setting_get --app=$app --key=node_id) - - +final_path=$(ynh_app_setting_get --app=$app --key=final_path) +virtualisation=$(ynh_app_setting_get --app=$app --key=virtualisation) +if [ "$virtualisation" = "true" ] +then + nbd_index=$(ynh_app_setting_get --app=$app --key=nbd_index) +fi #================================================= # REMOVE NODE CONFIGURATION #================================================= @@ -35,7 +38,7 @@ $final_path/garage -c $final_path/garage.toml layout remove "$node_id" apply_layout "$final_path/garage -c $final_path/garage.toml " if [ $? -ne 0 ] then - ynh_die --message="unable to remove the node. Maybe the number of node staying alive is not enough" + ynh_print_warn --message="unable to remove the node. Maybe the number of node staying alive is not enough" fi @@ -69,14 +72,16 @@ ynh_script_progression --message="Removing logrotate configuration..." --time -- ynh_remove_logrotate -#================================================= -# REMOVE VIRTUAL DISK -#================================================= -ynh_script_progression --message="umount virtual disk..." --time --weight=1 - -# Remove the app directory securely -$final_path/umount_disk.sh $nbd_index +if [ "$virtualisation" = "true" ] +then + #================================================= + # REMOVE VIRTUAL DISK + #================================================= + ynh_script_progression --message="umount virtual disk..." --time --weight=1 + # Remove the app directory securely + $final_path/umount_disk.sh $nbd_index +fi #================================================= # REMOVE APP MAIN DIR #================================================= diff --git a/scripts/restore b/scripts/restore index b2cef2c..9ef479e 100755 --- a/scripts/restore +++ b/scripts/restore @@ -31,6 +31,8 @@ app=$YNH_APP_INSTANCE_NAME domain=$(ynh_app_setting_get --app=$app --key=domain) final_path=$(ynh_app_setting_get --app=$app --key=final_path) datadir=$(ynh_app_setting_get --app=$app --key=datadir) +weight=$(ynh_app_setting_get --app=$app --key=weight) +virtualisation=$(ynh_app_setting_get --app=$app --key=virtualisation) #================================================= @@ -73,9 +75,18 @@ chown -R $app:$app "$final_path" #================================================= ynh_script_progression --message="Restoring the data directory..." --time --weight=1 -ynh_restore_file --origin_path="$datadir" --not_mandatory +mkdir -p "$datadir/data" + +if [ "$virtualisation" = "true" ] +then + # to be sure to not exceed size limit, i use a virtual disk with a fix size to have a max limit size. + qemu-img create -f qcow2 $datadir/garage_data.qcow2 "$weight"G + nbd_index=$($final_path/mount_disk.sh "$datadir" true) + ynh_app_setting_set --app=$app --key=nbd_index --value=$nbd_index +fi + +#ynh_restore_file --origin_path="$datadir/data" --not_mandatory -mkdir -p $datadir # FIXME: this should be managed by the core in the future # Here, as a packager, you may have to tweak the ownerhsip/permissions