####### # installing new O/S, F11 at bios screen will get boot-menu to pick up UEFI - usbkey media # at least use /home from previous boot ###### # add nvidia/extra repo ... software-properties-gtk # use previous install's conf export OD=/mnt/old_distro cp -f $OD/etc/sudoers.d/ddp /etc/sudoers.d ; # make sudo not ask for passwd for admin group chmod 440 /etc/sudoers.d/ddp #cp -f $OD/etc/rc.local /etc/rc.local ; # pwm fan control & start_vpn for pug AND emit_cards_up for mara cp -f $OD/etc/default/grub /etc/default/grub ; # GRUB_TIMEOUT=1, GRUB_BACKGROUND=/home/ddp/Pictures/star-for-grub.png ; GRUB_GFXMODE="auto" update-grub # in case old /home has diff uid for ddp chown -R ddp:ddp /home/ddp apt-get update apt-get -y install xfce4 xfce4-power-manager bind9 vim openssh-server mplayer \ unrar vsftpd nmap net-tools iftop samba ethtool gsmartcontrol imagemagick \ vlc nethogs tcl lm-sensors pavucontrol sysstat elinks smbclient whois \ ubuntu-restricted-extras mdadm ffmpeg bind9 faenza-icon-theme ldap-utils \ cairo-dock-core cairo-dock-plug-ins mkvtoolnix gnome-system-tools curlftpfs \ sensors-applet handbrake iotop hardinfo smem docker-compose nvme-cli \ libjpeg-turbo-progs dbus-broker python3.10-venv # python/flask... apt-get -y install python3-pip python3-psycopg2 libpq-dev gunicorn mediainfo cmake libgl1-mesa-glx libglib2.0-0 python3-ldap # do this as ddp: pip3 install flask flask_login flask-ldap3-login sqlalchemy flask-sqlalchemy SQLAlchemy-serializer marshmallow-sqlalchemy flask-marshmallow flask-wtf flask-bootstrap pymediainfo ExifRead opencv-python datetime pytz dlib face_recognition Werkzeug flask-compress # hddtemp (for now, does not seem to be in u 22.04), so get it here wget http://archive.ubuntu.com/ubuntu/pool/universe/h/hddtemp/hddtemp_0.3-beta15-54_amd64.deb apt install ./hddtemp* # test this rsync out, I cp'd last time and broke perms rsync -axvn $OD/srv/* /srv/ systemctl stop named systemctl disable named docker-compose -f /srv/docker/config/docker-compose.yml up -d pihole ### DNS (pihole in container) echo "its likely /etc/hosts already contains 127.0.1.1 mara.ddp.net -- change this to 192.168.0.2" echo "if pihole is up, bound explictily to 192.168.0.2:53, systemd-resolved will work, and with the direct 192.168.0.2 in hosts, docker dns will work too" cp -f $OD/etc/systemd/resolved.conf /etc/systemd/resolved.conf systemctl restart systemd-resolved # get mara to use ldap for auth, etc. docker-compose -f /srv/docker/config/docker-compose.yml up -d openldap sudo apt install libnss-ldap libpam-ldap ldap-utils nscd # answer: #1) ldap://192.168.0.2 #2) dc=depaoli,dc=id,dc=au #3) 3 #4) Yes #5) No #6) cn=admin,dc=depaoli,dc=id,dc=au #7) a_real_admin_pass_word_for_2o20 # add ldap to end of passwd group shadow in /etc/nsswich.conf sudo pam-auth-update ; # tick on home dir creation # update everything :^) apt-get -y dist-upgrade cp -f $OD/etc/apache2/ports.conf /etc/apache2/ports.conf systemctl restart apache2 # FIX /etc/fstab (likely this would work) -- relies on you already having installed with /home AND UUID/mount options not changing grep '/export' $OD/etc/fstab >> /etc/fstab grep '/backup' $OD/etc/fstab >> /etc/fstab # get the raid array back mdadm --assemble --scan mkdir /export mkdir /backup mount -a docker-compose -f /srv/docker/config/docker-compose.yml up # fix noisy logs: cp $OD/etc/systemd/system/run-docker-.mount.d/10-silence.conf /etc/systemd/system/run-docker-.mount.d/10-silence.conf systemctl daemon-reload # mara (nfs server) apt-get -y install nfs-kernel-server portmap cp $OD/etc/default/nfs-common /etc/default cp $OD/etc/default/nfs-kernel-server /etc/default cp $OD/etc/modprobe.d/nfs-lockd.conf /etc/modprobe.d/ cp -f $OD/etc/exports /etc/exports # mara (various) To note: upnp is off on the modem (so no internet), # but allowed from anything on 192.168 to 1900 (emby) and from 1900 on 192.168 (emby response) sudo ufw allow from 192.168.0.0/24 to any port 20 comment "ftp-data" sudo ufw allow from 192.168.0.0/24 to any port 21 comment "ftp" sudo ufw allow from 192.168.0.0/24 to any port 22 comment "ssh" sudo ufw allow from 192.168.0.0/24 to any port 53 comment "DNS" sudo ufw allow from 192.168.0.0/24 to any port 111 comment "portmap(nfs)" sudo ufw allow from 192.168.0.0/24 to any port 389 comment "ldap" sudo ufw allow from 192.168.0.0/24 to any port 445 comment "samba" sudo ufw allow from 192.168.0.0/24 to any port 1400 comment "hass/sonos discovery (192 range)" sudo ufw allow from 172.18.0.0/12 to any port 1400 comment "hass/sonos discovery (172 range)" sudo ufw allow from 192.168.0.0/24 to any port 1401 comment "hass/sonos discovery2 (192 range)" sudo ufw allow from 172.18.0.0/12 to any port 1401 comment "hass/sonos discovery2 (172 range)" sudo ufw allow from 192.168.0.0/24 to any port 1883 comment "mqtt/mosquitto broker" sudo ufw allow proto udp from 192.168.0.0/24 to any port 1900 comment "upnp/emby" sudo ufw allow proto udp from 192.168.0.0/24 port 1900 comment "upnp/response" sudo ufw allow from 192.168.0.0/24 to any port 2049 comment "nfs" sudo ufw allow proto tcp from 192.168.0.0/24 to any port 3306 comment "mysql" sudo ufw allow from 192.168.0.0/24 to any port 4045 comment "lockd(nfs)" sudo ufw allow from 192.168.0.0/24 to any port 4046 comment "statd(nfs)" sudo ufw allow from 192.168.0.0/24 to any port 4047 comment "mountd(nfs)" sudo ufw allow from 192.168.0.0/24 to any port 5000 comment "pa dev port (Flask)" sudo ufw allow from 192.168.0.0/24 to any proto udp port 5353 comment "mDNS - used by esphome/hass" sudo ufw allow from 192.168.0.0/24 to any port 5678 comment "apache on mara now" sudo ufw allow from 192.168.0.0/24 to any port 6543 comment "myth" sudo ufw allow from 192.168.0.0/24 to any port 6544 comment "myth (api)" sudo ufw allow proto udp from 192.168.0.0/24 to any port 3610 comment "echonet poll - dining-ac" sudo ufw allow proto udp from 192.168.0.0/24 to any port 7777 comment "ark server - game port" sudo ufw allow from 192.168.0.0/24 to any port 27015 comment "ark server - steam port" sudo ufw allow from 192.168.0.0/24 to any port 8080 comment "traefik (dashboard)" sudo ufw allow from 192.168.0.0/24 to any port 8096 comment "emby" sudo ufw allow from 192.168.0.0/24 to any port 8123 comment "hass" sudo ufw allow from 192.168.0.0/24 to any port 6052 comment "esphome dashboard" sudo ufw allow from 192.168.0.0/24 to any proto tcp port 10090:10092 comment "vsftpd-passive" sudo ufw deny 1900 comment "block UPnP" # mail sudo ufw allow proto tcp to any port 25 comment "open to world: mail/smtp for mail.depaoli.id.au" sudo ufw allow proto tcp to any port 80,443,465,587,993 comment "open to world: 80 (lets enc renewal), webmail/others (443), smtps-ssl (465), smtps (587), imaps (993)" # docker networks (not sure this is possible or needed without the static route)? sudo ufw allow from 172.16.0.0/12 comment "allow docker networks to talk back to mara" sudo ufw enable sudo reboot; # all but myth should work on a reboot, so lets test... cp $OD/etc/samba/smb.conf /etc/samba/smb.conf # echo need to set up users in samba with (needs local account for now) sudo smbpasswd -a ddp ####### TODO: sudo smbpasswd -a mandy sudo smbpasswd -a cam ####### END TODO # if on mara do more... apt-get -y install mythtv mythweb # for myth... # start with h/w firmware, still need this I believe: sudo add-apt-repository ppa:b-rad/kernel+mediatree+hauppauge sudo apt-get install linux-firmware-hauppauge # vi /etc/passwd ; # make mythtv - 500:500 # vi /etc/group ; # make mythtv - 500 echo "find the current mythtv: uid and gid -- untested below here" myth_uid=`id mythtv | cut -f2 -d= | cut -f1 -d'('` myth_gid=`id mythtv | cut -f3 -d= | cut -f1 -d'('` chown -R mythtv:mythtv /home/mythtv echo "is this still needed, I've moved my logs to /var/tmp ???" #chown -R mythtv:mythtv /var/log/mythtv/ find / -uid $myth_uid --ls ; # should be none left, but hey if there is some, then do this: # find / -uid $myth_uid --exec chown mythtv:mythtv {} \; find / -gid $myth_gid --ls ; # should be none left, but hey if there is some, then do this BUT I DONT KNOW HOW, so look before you do ### this would imply a file owned by someone other than mythtv, but with a group of mythtv... # find / -uid $myth_gid --exec chgrp mythtv {} \; # reset myth's pwd in mysql to mythtv mysql -u root mysql mysql> alter user mythtv identified by 'mythtv'; #mysql> grant all on mythconverg.* to mythtv@'192.168.0.%' identified by 'mythtv'; mysql> FLUSH PRIVILEGES; mysql> quit # let other boxes (actually including mara if you use --host) have access ## during install of myth backend, I said yes to other boxes, yes to mythweb only and no to a password # if mariadb: "change bind_address to 0.0.0.0 in /etc/mysql/mariadb.conf.d/50-server.conf" # if mysql: #echo "need to set bind_address = 0.0.0.0 in /etc/mysql/mysql.conf.d/mythtv.cnf" cp $OD/etc/mysql/mysql.conf.d/mythtv.cnf /etc/mysql/mysql.conf.d/mythtv.cnf #### mythtv needs password change echo "need to change 'setenv db_password mythtv' in ./apache2/sites-available/mythweb.conf" ############ TEST FIRST ################ echo "can we login from root account locally" sudo bash mysql --user=mythtv --password=mythtv mythconverg exit echo "can we login from ddp account locally" mysql --user=mythtv --password=mythtv mythconverg echo "can we login from ddp account via host" mysql --host=mara.ddp.net --user=mythtv --password=mythtv mythconverg # to refresh content to previous mysql -u root -p mythconverg (enter the password you just set above when prompted) mysql> source /home/ddp/installed/tv/mythconverg.dump mysql> quit ##### may need to log out and back in for group to just work or just run mythfrontend and say yes cp -r $OD/usr/share/mythtv/mythweather/scripts/bom* /usr/share/mythtv/mythweather/scripts # force pwd in config to be mythtv cp $OD/etc/mythtv/config.xml /etc/mythtv/config.xml ### need to ensure tuner card is 'up' before mythtv-backend: cp $OD/etc/udev/rules.d/99-mythbackend.rules /etc/udev/rules.d/99-mythbackend.rules ; # makes udev send a systemd ??? for a dvb device ### diff these 2 files, we need the '4 lines - 2 comments and the Requires/After lines for dev-dvb-* to be in the [Unit] stanza #### TODO: these were lost on the 20.04 install/upgrades its seems... (maybe just not needed anyway) diff $OD/etc/systemd/system/multi-user.target.wants/mythtv-backend.service /etc/systemd/system/multi-user.target.wants/mythtv-backend.service cp $OD/etc/systemd/system/multi-user.target.wants/mythtv-backend.service /etc/systemd/system/multi-user.target.wants/mythtv-backend.service # myth cron's cp -rf $OD/etc/cron.myth /etc cp -f $OD/etc/crontab /etc # copy over tv-icons, can get a new set from: https://pureservices.com.au/our-work/australian-tv-logos-icons-tvheadend-kodi/ cp -f $OD/var/cache/mythweb/image_cache/* /var/cache/mythweb/image_cache # renew all our certs... cp -f $OD/etc/cron.weekly/letsencrypt-cert-renew /etc/cron.weekly/ cp -f $OD/etc/cron.daily/record-docker-updates /etc/cron.daily cp -f $OD/etc/cron.daily/clean-up-old-docker-images /etc/cron.daily # myth logs: #for i in ddp.log fill.cron front.log mythbackend.log ; do sudo touch /var/log/mythtv/$i; done #chown -R mythtv:mythtv /var/log/mythtv #chmod 664 /var/log/mythtv/* #chmod 2775 /var/log/mythtv # shepherd dependencies -- used to need: libgetopt-mixed-perl apt-get -y install xmltv libxml-simple-perl libalgorithm-diff-perl libdata-dumper-simple-perl \ libdate-manip-perl liblist-compare-perl libdatetime-format-strptime-perl \ libhtml-parser-perl libxml-dom-perl libgd-gd2-perl \ libarchive-zip-perl libio-string-perl libdbi-perl libsort-versions-perl libfile-find-rule-perl ln -s /home/ddp/.shepherd/shepherd /usr/bin/tv_grab_au apt-get -y purge mythtv-dbg apt-get -y purge modemmanager apt-get -y purge avahi-daemon apt-get -y purge speech-dispatcher apt-get -y autoremove # UPS (powerpanel) dpkg -i /home/ddp/installed/debs/powerpanel* # test ups... sudo /usr/sbin/pwrstat -status # reduce # of apache workers for this box cp $OD/etc/apache2/mods-available/mpm_prefork.conf /etc/apache2/mods-available/mpm_prefork.conf echo "You need to:" echo " tweak httpds to 1 (/etc/apache2/mods-enabled/mpm_prefork.conf)" echo " follow ~ddp/src/LIBRARY/README ; restart postgresql/apache2 to make sure all conf changes take" echo " configure printer (color) - find in the gui just works" echo "if on mara reboot to get the tuners to work" # for prettiness :^) #apt-get -y install plymouth-theme* #update-alternatives --config default.plymouth ; # choose solar #modify /etc/default/grub to have: # GRUB_GFXPAYLOAD_LINUX=auto #echo "FRAMEBUFFER=y" > /etc/initramfs-tools/conf.d/splash #update-initramfs -u # get cool gnome bg #cp $OD/usr/share/backgrounds/gnome-step-into-freedom.jpg /usr/share/backgrounds/gnome-step-into-freedom.jpg # still trying to get better gnome3 experince... # add-apt-repository ppa:webupd8team/gnome3 # apt-get update # apt-get -y install gnome-shell-extensions-autohidetopbar # want newer nvidia? # add-apt-repository ppa:ubuntu-x-swat/x-updates # want bleeding edge nvidia? # add-apt-repository ppa:xorg-edgers/ppa # femon for dvb debugging apt-get -y install dvb-apps ### tv headend sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 379CE192D401AB61 echo "deb https://dl.bintray.com/tvheadend/deb xenial stable" | sudo tee -a /etc/apt/sources.list sudo apt update sudo apt-get install tvheadend # ensure wol works in new systemd world conn=`nmcli -f NAME con show -a | tail -n1 | xargs` sudo nmcli c modify "$conn" 802-3-ethernet.wake-on-lan magic # docker... # keep log files to a small size cp $OD/etc/docker/daemon.json /etc/docker/daemon.json cp $OD/etc/cron.myth/cron.weekly/letsencrypt-cert-renew /etc/cron.myth/cron.weekly/letsencrypt-cert-renew # hack to stop systemd-sleep? systemctl mask sleep.target suspend.target hibernate.target hybrid-sleep.target ### BOOK DB apt-get -y install postgresql php php-pgsql php-gd php-pear php-db libpgtcl bwidget \ tcllib libtext-csv-xs-perl libwww-perl libdbi-perl libdbd-pg-perl apache2 a2dismod mpm_event a2enmod php7.2 systemctl restart apache2 # for node (for library) apt install -y npm # final setup echo "follow ~ddp/src/LIBRARY/README" # private git server sudo apt install git # ONE-OFF: did this to get home dir setup to take my ssh key and have a bare git repo ssh git@localhost mkdir -p ~/.ssh && chmod 0700 ~/.ssh touch ~/.ssh/authorized_keys && chmod 0600 ~/.ssh/authorized_keys exit sudo cp ~/.ssh/id_rsa.pub /home/git/.ssh/authorized_keys sudo su - git git init --bare ~/docker-configs exit cd /export/docker/config/ git init . git add . git commit -m 'initial commit of larger docker-compose file including portainer, plex & phpldapadmin that I have removed to keep mara running better for now' git remote add origin git@192.168.0.2:docker-configs git push -u origin master ### END ONE-OFF ### # want to get a copy from git? git clone git@192.168.0.2:docker-configs or git@192.168.0.2:pybook ### ### ODD FREEZING? #### - at least put that if I hit power button, it will shutdown, hopefulyl can catch last real log line? - also altered sudo vi /etc/default/acpi-support # and then set SUSPEND_METHODS="none" - sudo modprobe softdog - sudo service watchdog start -- edited /etc/watchdog.conf ### RAID: (here, sdd is an existing raid disk, and sde (external) is being added sudo parted /dev/sdd print ; # of an existing disk to see what we want (likely a single partition aligned for performance - so starting not at 0)K sudo parted /dev/sde print sudo parted /dev/sde > mklabel gpt > mkpart primary 2048s 100% > quit sudo mdadm /dev/md0 --add /dev/sde1 sudo mddm -D /dev/md0 ; # check we now have a 3rd device, no resyncing going on, its just a hot spare sudo mdadm /dev/md0 --grow --raid-devices=3 sudo mdadm -D /dev/md0 ; # now we should see a 3-way mirror, resync occurring, wait hours :( sudo mdadm /dev/md0 --fail /dev/sdc1 ; # or whichever is your old disk (can use smartctl or parted to wokr it out) sudo mdadm /dev/md0 --grow --raid-device=2 sudo mdadm -D /dev/md0 ; # now we should see a 2-way mirror, resync finished! # probably best to do this offline. (and needs to be finalised/validated) SO... single user, sudo umount /export sudo mdadm /dev/md0 ??? (grow raid) sudo ext2resizefs? /dev/md0??? (grow fs) sudo mount /dev/md0 /export df -h /export ; # should now have an online larger disk / filesystem :) sudo mdadm -D /dev/md0 ; # not sure if it will now be resyncing the additional space or not? # update copy of restic for backup/restore: sudo su - restic ~/bin/restic self-update ; # needed as the version in the repo was quite old ### sublime wget -qO - https://download.sublimetext.com/sublimehq-pub.gpg | sudo apt-key add - sudo apt-get install apt-transport-https echo "deb https://download.sublimetext.com/ apt/stable/" | sudo tee /etc/apt/sources.list.d/sublime-text.list sudo apt-get update sudo apt-get install sublime-text # restrict journald log size to 1G (it will be 4G if we leave defaults) # in /etc/systemd/journald.conf: # SystemMaxUse=1G # odd issue/slowness? in udev as this is trying to deal with mythtv/firewire, I # have no firewire, so ditch it: sudo rm /usr/lib/udev/rules.d/41-mythtv-permissions.rules