# Demo using Windows 10
# using git 2.31.1 bash terminal
# https://github.com/git-for-windows/git/releases/download/v2.31.1.windows.1/Git-2.31.1-64-bit.exe


# Demo 1
# Log into GitLab
# Go to lab repo: https://gitlab.liu.se/olaha93/bigdata 
# Fork repository to yours
# Change repository to private
# Invite Teachers, Lab assistant and Lab partner
# Done

# Demo 2 SSH and GIT
# GIT comes installed on Linux
# Windows and MAC: Install

# Set up SSH key 
# Generate ssh key:
ssh-keygen 

# Go to folder and open id_rsa.pub
# Copy public key to GitLab

# Copy SSH Key to GitLab

# Clone repositories to local machine
git clone ... (SSH or HTTPS)

# Demo 3 - Connecting to Sigma
ssh x_erros@sigma.nsc.liu.se

# -X : allow X11-forwarding (if configured)
ssh -X x_erros@sigma.nsc.liu.se


# Print directory contents
ls

# Create folder
mkdir bigdata2021

# Show new folder exists
ls

# Set up ssh to sigma
ssh-copy-id sigma_user@sigma...

# Change directory to bigdata2021
cd bigdata2021

# Clone repo to sigma
git clone ... (SSH or HTTPS)

# Copy demo
# cp FROM_PATH TO_PATH
# To copy folder, use -r or -R
cp -R /software/sse/manual/spark/BDA_demo ~/bigdata2021/

# Copy data (.csv files) to (for example) Documents folder
cp -R /software/sse/manual/spark/BDA_demo/input_data ~/Documents

# To leave use : exit
exit

# Demo 3.5 ThinLinc
# ThinLinc

sigma.nsc.liu.se
# Use sigma username
# Use sigma password

# For LiU
thinlinc.edu.liu.se
# Use liuID as username
# Use liuID password

# Remember to log out

# Demo 4 Develop
# Copy demo.py and run... .q file to local computer
scp x_erros@sigma.nsc.liu.se:bigdata2021/BDA_demo/demo.py ./
scp x_erros@sigma.nsc.liu.se:bigdata2021/BDA_demo/run_yarn.q ./

# Create lab1.py file by renaming demo.py
# Modify lab1.py
# Create run.q : by renaming run_yarn.q
# Modify run.q : directory of input data 
# Change data path to match data in Documents
# Upload it to GitLab
# cd to folder with files or add path

git add lab1.py run_yarn.q
git commit -m "Added lab1.py and run_yarn.q"
git push

# Alternative
# Copy file from local machine to user on Sigma
# Open terminal on local machine
# scp FROM_PATH TO_PATH
scp ./run_yarn.q x_erros@sigma.nsc.liu.se:bigdata2021/SSH/bigdata/lab1-spark

# Log in to Sigma
ssh ...

# Change directory to where cloned directory is
cd ...

# Pull changes
git pull


# Demo 4 Submit a job
# While in same folder as the demo run_yarn.q file 
# Alternative use modified lab1.py / run.q files 
# Slurm documentation : https://slurm.schedmd.com/documentation.html
# -A : Account
# --reservation : Reservation
# Running demo
sbatch -A liu-compute-2021-4 --reservation devel run_yarn.q

# If lab session use liu-compute-2021-4 as reservation
sbatch -A liu-compute-2021-4 --reservation liu-compute-2021-4 run_yarn.q

# Look at queue
squeue -u x_erros

# Copy output from home/x_erros/... on Sigma to (current) local folder
# Using local terminal
# * means all files
scp -r x_erros@sigma.nsc.liu.se:bigdata2021/demo/BDA_demo/output/* ./


# Other fun things
# use Putty and Xming to x-forward firefox

# Check OS used:
cat /etc/os-release

# Set up ssh on Sigma
ssh-keygen
# Open id_rsa.pub using VIM or Emacs
# Copy public key to GitLab