remove bom

awk '{ if (NR==1) sub(/^\xef\xbb\xbf/,""); print }' INFILE > OUTFILE




sed -i '1 s/^\xef\xbb\xbf//' *.txt




display binary file

xxd 




tail -c +4 UTF8 > UTF8.nobom

# awk '{if(NR==1)sub(/^\xef\xbb\xbf/,"");print}' text.txt
# sed -e '1s/^\xef\xbb\xbf//' text.txt
# tail --bytes=+4 text.txt
http://stackoverflow.com/questions/1068650/using-awk-to-remove-the-byte-order-mark