====== Awk Command tips ======
==== multiple field separators ====
awk 'BEGIN { FS="[/()\" ]" } {print $1 " " $2 " " $3 }' data.txt
==== printing quotes ====
awk -v q="'" '{print "value is" q $2 q}' inputfile
or
awk '{print "value is \x27" $2 "\x27"}' inputfile
or
awk '{ print "value is","'\''" $2 "'\''" }' input.txt
==== multiple field separators ====
http://bashshell.net/utilities/using-variables-with-awk
grep -i brown /var/log/maillog |awk -F'[<>]' '/whitelisted/ {print $4}' |sort |uniq -c
==== Filter on column ====
tail -100 /var/log/httpd/access_log |awk -F '[ "]' '$11~/20[06]/ {print $8,$11,$12}'
tail -100 /var/log/httpd/access_log |awk -F '[ "]' '$11~/20[06]/ {sum +=$12;print $8,$11,$12} END {print sum/1024/1024 "MB"}'
==== Count web hits ====
http://www.unix.com/shell-programming-scripting/65529-using-uniq-awk.html
cat /var/log/httpd/access_log |awk -F '[ "]' '$11~/200/ {hits[$8]++} END {for (i in hits) print hits[i], i}' |sort -n
==== udp firewall hits on athena ====
watch 'grep DROPPED /var/log/messages |grep -v 0.0.0.0|grep -v SRC=128.173|grep UDP|grep -v DPT=137|grep -v DPT=67|grep -v DPT=17500|tail'
pick a udp port being hit. e.g., 56846 Place a sniffer on the port:
tcpdump -i eth1 -vnn -s0 -X port 56846 -w port56846
tcpdump -nn -v -s0 -X -r port56846 |less
collect the addresses from the logs:
awk -F '[ =]' '/56846/ {print $14}' /var/log/messages >>athena-udp-src
==== Dropbox machines ====
awk 'BEGIN { FS="[ =]" } /DPT=17500/ && /^Jun 21/ {print $14}' /var/log/messages |sort -n |uniq |while read line; do echo -n $line " "; host $line ;done
Be sure to change the date from Jun 21 to your desired date
==== bootp UDP from 0.0.0.0 addresses ====
awk -F ":" '/FIREWALL/ && /DPT=67/ && /SRC=0.0.0.0/ {print $10":"$11":"$12":"$13":"$14":"$15}' /var/log/messages |sort|uniq -c|sort -n
==== print out fileserver volume information ====
lvdisplay |awk '/LV Name/ || /LV Size/ || /VG Name/ || /Block device/ {print $0}'
on one line:
lvdisplay |awk '/LV Name/ {NAME=$3} /LV Size/ {SIZE[NAME]=$3 ; } END {for (x in SIZE) print x,SIZE[x]}'
Add on the mountpoint found in /etc/fstab
lvdisplay | cat - /etc/fstab |awk '/LV Name/ {NAME=$3} /VG Name/ {VG[NAME]=$3 } /LV Size/ {SIZE[NAME]=$3$4} /Block device/ {DEV[NAME]=$3} /ext3/ {for (i in SIZE) {if (i == $1) {MOUNT[i]=$2}}} END {for (x in SIZE) print VG[x],SIZE[x],DEV[x],x,MOUNT[x]}' |sort
format for the wiki (for non-mapper version of /etc/fstab names and ext3)
lvdisplay | cat - /etc/fstab |awk '/LV Name/ {NAME=$3} /VG Name/ {VG[NAME]=$3 } /LV Size/ {SIZE[NAME]=$3$4} /Block device/ {DEV[NAME]=$3} /ext3/ {for (i in SIZE) {if (i == $1) {MOUNT[i]=$2}}} END {for (x in SIZE) print "| | |"VG[x]"| |"SIZE[x]"|"DEV[x]"|"x"|"MOUNT[x]"|"}' |sort
==== create links to grad or FS entries ====
[root@alexandria ~]# cat linkPartitions.awk
BEGIN{
FS = "/"
}
{
print "ln -s /home/"$6"/"substr($4,0,length($4)-1)"/ /home/"$3"/"substr($4,0,length($4)-1)
}
==== Check /etc/hosts for inconsistent host entries ====
awk '/^128/ {FS="[. ]"; if ($5!=$9) print $5,$9}' /etc/hosts
==== list members of group with pid and name ====
Run this from a machine using sssd with ldap (like montgolfier) instead of nis because nis does not give the full name.
getent passwd `getent group |grep grad-lab7-clipper: | cut -d : -f 4| tr , " "|sort -n` |awk 'BEGIN { FS = ":" } ; { print $1,$5 }' |sort -n
==== computer audit list ====
http://www.theunixschool.com/2012/05/awk-join-or-merge-lines-on-finding.html
partial solution:
ldapsearch -H ldaps://neptune.aoe.vt.edu -Y GSSAPI -N -b 'dc=aoe,dc=vt,dc=edu' "(objectclass=Computer)" name operatingSystem operatingSystemVersion operatingSystemServicePack |awk '/name:/ || /operating/ {print $0}'|awk '/name:/ {if (x)print x;x="";}{x=(!x)?$2:x","$2;}END{print x;}'
ldapsearch -H ldaps://neptune.aoe.vt.edu -Y GSSAPI -N -b 'dc=aoe,dc=vt,dc=edu' "(objectclass=Computer)" name operatingSystem operatingSystemVersion operatingSystemServicePack |grep name: |awk '{print $2}' |while read line; do echo -n $line " "; host $line ;done |grep "has address"|sort -n|awk '{print $5,$1}'
complete solution:
ldapsearch -H ldaps://neptune.aoe.vt.edu -Y GSSAPI -N -b 'dc=aoe,dc=vt,dc=edu' "(objectclass=Computer)" name operatingSystem operatingSystemVersion operatingSystemServicePack |awk '/name:/ || /operating/ {print $0}'|awk '/^name/ {FS=":";if (x)print x;x="";}{x=(!x)?$2:x","$2;}END{print x;}' |while read line; do host `echo -n $line|awk 'BEGIN { FS=","}{print $1;}'`|awk ' {if ($3 == "not") printf "%s,","none"; else printf "%s,",$4}';echo $line ;done
Complete solution with lastLogonTimestamp
ldapsearch -H ldaps://neptune.aoe.vt.edu -Y GSSAPI -N -b 'dc=aoe,dc=vt,dc=edu' "(objectclass=Computer)" name operatingSystem operatingSystemVersion operatingSystemServicePack lastLogonTimestamp |awk '/name:/ || /operating/ || /Logon/ {print $0}'|awk '/^name/ {FS=":";if (x)print x;x="";}{x=(!x)?$2:x","$2;}END{print x;}' |sort -n|while read line; do host -t A `echo -n $line|awk 'BEGIN { FS=","}{print $1;}'`|while read line2 ; do echo -n $line2 |awk ' {if ($3 == "not") printf "%s,","none"; else printf "%s,",$4}';echo $line ;done ;done > dns_computers.txt
To convert the timestamp in excel:
F2=lastLogonTimestamp
http://myserverstuff.blogspot.com/2009/03/csvde-to-excel-human-readable-lastlogon.html
=IF(F2>0,F2/(8.64*10^11) - 109205,"")
To convert the timestamp in Linux (gives UTC):
http://meinit.nl/convert-active-directory-lastlogon-time-to-unix-readable-time
lastLogonTimestamp=130002228839738710
date -d "1970-01-01 `echo $(((130002228839738710/10000000)-11644473600))` sec GMT"
Windows tips:
http://blogs.technet.com/b/askds/archive/2009/04/15/the-lastlogontimestamp-attribute-what-it-was-designed-for-and-how-it-works.aspx
==== Linux machine address and version ====
ls -1 |while read line; do echo -n $(host -t A $line | cut -d " " -f 4 ) ; echo -n ",";echo -n $line"," ; echo -n $(ssh -o ConnectTimeout=2 $line 'cat /etc/redhat-release' < /dev/null);echo -n ","$(ssh -o ConnectTimeout=2 $line 'uname -r' < /dev/null) ;echo ;done 2> /dev/null > ~/sandbox/dns_computers_linux.txt
==== Who has used Tecplot ====
echo " $( ssh licenseserver2 'cat /opt/tecplot/rlm/teclmd.log' ) " | awk '/OUT/ {print $8}' | awk -F "@" '{print $1}'| sort | uniq | while read line; do echo -n $line " "; getent passwd | grep $line ;done | awk 'BEGIN { FS=":" } { print $5 }'
==== Comsol FlexLM license file parser ====
cat License43b-Combined.dat.stripped | perl -p -e 's/\\\r\n//' | grep -v FEATURE | grep -v ^# | grep -v ^SERVER | grep -v ^USE | grep -v VENDOR | awk '{ sub("\r$", ""); print }' | grep -v ^$ | tr -d '\011' | awk '{print $2,$4,$6,$8,$9,$11}' | column -t | sort > licensesort
==== create links in /home/grad ====
cd /home/grad
find /home/grad[1-5] -maxdepth 1 -mindepth 1 -type d ! -user root -printf "%p " -printf "%f\n" | xargs -L1 ln -s
cd /home/facultystaff
find /home/facultystaff[1-4] -maxdepth 1 -mindepth 1 -type d ! -user root -printf "%p " -printf "%f\n" | xargs -L1 ln -s
==== Allocated quota ====
repquota /home/facultystaff1 | grep ^# | awk '{qsum+=$5} END {print qsum}' | { read test; echo $(( $test / 1024 /1024 )); }
==== ossec ====
usernames tried from CCDFS1
awk -F ': ' '/CCDFS1/ {print $10}' ossec-archive-01.log
usernames tried from CCDFS1 with FAILURE in entry
awk -F ': ' '/CCDFS1/ && /FAILURE/ {print $10}' ossec-archive-01.log
unique usernames with count tried from CCDFS1 with FAILURE in entry
awk -F ': ' '/CCDFS1/ && /FAILURE/ {print $10}' ossec-archive-01.log | sort | uniq -c | sort -n
...more specific
awk -F ': ' '/CCDFS1/ && /AUDIT_FAILURE\(4776\)/ {print $12}' ossec-archive-01.log | sort | uniq -c
...now only if error code equals C00006a
awk -F ': ' '/CCDFS1/ && /AUDIT_FAILURE\(4776\)/ && $12 == "0xc000006a" {print $10}' ossec-archive-01.log | sort | uniq -c
failures not from CCDFS1
awk -F ': ' '! /CCDFS1/ && /AUDIT_FAILURE\(4776\)/ {print $0}' ossec-archive-01.log
logons (4776) non domain or non Kerberos
awk -F ': ' '! /CCDFS1/ && /\(4776\)/ {print $3,$7,$10,$11,$12}' ossec-archive-01.log | sort | uniq -c | sort -n
Failed Kerberos
awk -F ': ' '/\(4771\)/ {print $7,$10,$11,$12,$13,$14,$15}' /var/ossec/logs/archives/2013/Dec/ossec-archive-02.log | sort | uniq -c | sort
Remote Logins
awk -F ': ' '/\(4624\)/ && $14 == " 10 New Logon" {print $5}' /var/ossec/logs/archives/2013/Dec/ossec-archive-03.log | sort | uniq -c
awk -F ': ' '/\(4624\)/ && $14 == " 10 New Logon" {print $5,$7}' /var/ossec/logs/archives/2013/Dec/ossec-archive-04.log | sort | uniq -c | sort -n
awk -F ': ' '/\(4624\)/ && $14 == " 10 New Logon" {print $1,$5,$7}' /var/ossec/logs/archives/2013/Dec/ossec-archive-04.log
local logins
awk -F ': ' '/\(4624\)/ && $14 == " 2 New Logon" {print $1,$5,$7}' /var/ossec/logs/archives/2013/Dec/ossec-archive-06.log
zcat /var/ossec/logs/archives/2013/Dec/ossec-archive-05.log.gz | awk -F ': ' '/\(4624\)/ && $14 == " 2 New Logon" {print $1,$5,$7}'
Login types
awk -F ': ' '/\(4624\)/ {print $14}' /var/ossec/logs/archives/2013/Dec/ossec-archive-03.log | sort | uniq -c
Login types:
Logon Type Description
2 Interactive (logon at keyboard and screen of system)
3 Network (i.e. connection to shared folder on this computer from elsewhere on network)
4 Batch (i.e. scheduled task)
5 Service (Service startup)
7 Unlock (i.e. unnattended workstation with password protected screen saver)
8 NetworkCleartext (Logon with credentials sent in the clear text. Most often indicates a logon to IIS with "basic authentication") See this article for more information.
9 NewCredentials such as with RunAs or mapping a network drive with alternate credentials. This logon type does not seem to show up in any events. If you want to track users attempting to logon with alternate credentials see 4648.
10 RemoteInteractive (Terminal Services, Remote Desktop or Remote Assistance)
11 CachedInteractive (logon with cached domain credentials such as when logging on to a laptop when away from the network)
ipSec probes:
zcat *.log.gz | awk -F ': ' '/AUDIT_FAILURE\(4963\)/ {print $9}' | cut -d " " -f 1 | sort -n | uniq -c
or
cat ossec-archive-12.log | awk -F ': ' '/AUDIT_FAILURE\(4963\)/ {print $9}' | cut -d " " -f 1 | sort -n | uniq | xargs -L1 host
==== Multiple multi character field separators ====
Return dropped addresses in courier's iptables config file
grep DROP /etc/sysconfig/iptables | grep -v LOG | awk -F'-s | -j' ' /-s/ {print $0}'
awk will "grep" for lines with -s to avoid the "-j DROP" line at the end of the config file.
==== where are export folders mounted ====
ls -1 | while read line; do echo ; echo -n $line " "; df -Ph $line ;done
==== ossec logon id's ====
zgrep -i "AUDIT_SUCCESS(4672)" ossec-archive-24.log.gz | grep -v "Logon ID:[[:space:]]*0x0"| awk -F ": " '{print $13}' | awk '{print $1}' | sort -n | uniq -c | grep bad
==== quota sort with percentages and divide by zero detection ====
repquota /export/facultystaff[1-4] | grep -v ^\- | grep -v ^User | grep -v ^root | grep -v ^Block | grep -v ^\* | grep -v ^" " | grep -v ^$ | awk '{if( $5 + 0 != 0) print $3/1024/1024,$5/1024/1024,$1,$3/$5; if( $5 + 0 == 0) print $3/1024/1024,$5/1024/1024,$1,0}' | awk '{printf "%4.1f\t", $1;printf "%4.1f\t", $2;printf "%3.1f%\t", $4;printf "%s\n",$3; }' | sort -nr | awk '{print $4,"\t"$1,"\t"$2,"\t"$3 }'| column -t
repquota /export/facultystaff[1-4] | grep -v ^\- | grep -v ^User | grep -v ^root | grep -v ^Block | grep -v ^\* | grep -v ^" " | grep -v ^$ | awk '{if( $5 + 0 != 0) print $3/1024/1024,$5/1024/1024,$1,100*($3/$5); if( $5 + 0 == 0) print $3/1024/1024,$5/1024/1024,$1,0}' | awk '{printf "%4.1f\t", $1;printf "%4.1f\t", $2;printf "%3.1f%\t", $4;printf "%s\n",$3; }' | sort -nr | awk '{print $4"\t"$1"\t"$2"\t"$3 }'
==== /etc/project to /etc/projid ====
awk -F[/:] '!/^($|[[:space:]]*#)/ {print $NF":"$1}' /etc/projects >> /etc/projid
This ignores comments and blank lines, separates the fields with either / or : and reorders the output
==== extract file extension from tivoli report ====
cat tivoli-errors | grep "Object increased in size during compression" | cut -d" " -f 9- | sed 's/[(][^)]*)//g' | awk -F"/" '{print $(NF-1)"-"$NF}' | grep -E "\." | awk -F. '{print $NF}' | sort | uniq
==== find newest 10 files ====
find . -type f -printf "%C@ %p\n" | sort -rn | head -n 10 | cut -d\ -f2- | awk '{print "\""$0"\""}'| xargs -L1 ls -Fla
====== find ======
http://www.unix.com/unix-for-dummies-questions-and-answers/50465-create-list-files-were-modified-after-given-date.html
ThobiasVakayil ThobiasVakayil
"-atime/-ctime/-mtime" the last time a files's "access time", "file status" and "modification time", measured in days or minutes. Time interval in options -ctime, -mtime and -atime is an integer with optional sign.
* n: If the integer n does not have sign this means exactly n days ago, 0 means today.
* +n: if it has plus sing, then it means "more then n days ago", or older then n,
* -n: if it has the minus sign, then it means less than n days ago (-n), or younger then n. It's evident that -1 and 0 are the same and both means "today".
Examples:
* Unordered List ItemFind everything in your home directory modified in the last 24 hours:
find $HOME -mtime 0
* Find everything in your home directory modified in the last 7 days:
find $HOME -mtime -7
* Find everything in your home directory that have NOT been modified in the last year:
find $HOME -mtime +365
* To find html files that have been modified in the last seven days, I can use -mtime with the argument -7 (include the hyphen):
find . -mtime -7 -name "*.html" -print
*If you use the number 7 (without a hyphen), find will match only html files that were modified exactly seven days ago:
find . -mtime 7 -name "*.html" -print
* To find those html files that I haven't touched for at least 7 days, I use +7:
find . -mtime +7 -name "*.html" -print
* Find files on specific date
find . -newermt 2013-03-26 ! -newermt 2013-03-27