Shell Tips

Array Operations

#Append
var=( ${array[@]-} $(echo "$variable") )

#Get Unique elements from any array
echo ${oldArray[@]}|awk '{for(i=1;i<NF;i++) printf "%s\n",$i}' | sort -u

#Check if element is present in an array

File Operations

#list files greater than 10bytes
find *.csv.gz -type f -size +10

#list files less than 11 bytes
find *.csv.gz -type f -size -11

#add line number
cat -n infile > outfile

#fetch nth line. Example fetch 74th line from file
awk 'NR==74' file
head -n 74 file | tail -1 

#fetch nth line from all the files matching certain criteria
find path_to_folder -n "file_pattern_* | xargs -n 1 awk 'NR==74' 

#add two files column wise using tab as delimiter
paste -d$'\t' file1 file2 > newfile

# remove ctrl-M from end of line. Ctrl-M is dos format to indicate end of line. Use tr command to remove it
cat file | tr -d '\015' > newfile

#change delimiter from tab to pipe
cat file | tr "\t" '|' > newfile

# Remove lines present in file 1 and file 2
cat file1 file2 | sort | uniq -u > newfile

#Read file line by line
while read line
do
  echo $line
done < input_file

#change file extension from gz to txt
for file in folder/*.gz; do mv $file `echo $file | sed 's/\(.*\.\)gz/\1txt/'`; done

Postgres Command Line Utilities

#Delete all rows from all tables. In xargs command -n1 indicates that process each line separately and -I table refers to the table.
psql -U user database -c "select relname, n_live_tup from pg_stat_user_tables order by n_live_tup desc;" | awk -F'|' '{if($2 > 0) print $1}' | xargs -n1 -I table psql -U user database -c "delete from table;"

#display row count for all tables in a database
psql -U user database -c "select tablename from pg_tables where tablename not like 'pg_%' and tablename not like 'sql_%';" | xargs -n1 -I table psql -U user database -c "select 'table' as table, count(*) from table;"

No related posts.