#!/bin/bash
#
# MAGY (Microsoft, AOL, Google, Yahoo) Consumer Domains Generator v1.9
# by Al Iverson - aiverson AT wombatmail.com 9/7/2021
#
# This script will check DNS to look for all possible known consumer webmail domains
# Owned by Google, Microsoft, or Yahoo (Yahoo/AOL/Verizon)
# MTA admins may find it useful for purposes of throughput/throttle settings
# This script does not intentionally include corporate domains; only consumer mailbox domains
#

function pick_server {

#
#I was going to have multiple DNS servers and have it pick at random
#But it was overkill, so I removed it
echo "8.8.4.4"

}

#
#Note the starting time, in readable date/time
#and seconds since epoch
STARTDAT=`date "+%s"`
STARTDAT2=`date`
DATE3=`date "+%D"`

cat << EOF
Starting at: $STARTDAT2
EOF

#
#Create CSV, add header row
OUT="magy-domains.csv"
echo "ISP,Domain,CC/TLD,Country,Region,Example MX,Notes (Generated on $DATE3)" > $OUT

#
#First, let us manually start with Google
#Remember that Google had "googlemail.com" for .de users from 2004-2012
#Those addresses are still valid
DSERVER=`pick_server`
GOO1=`host -t mx gmail.com $DSERVER | grep " is handled " | cut -d" " -f6- | sort | tail -1 | cut -d" " -f2`
GOO2=`host -t mx googlemail.com $DSERVER | grep " is handled " | cut -d" " -f6- | sort | tail -1 | cut -d" " -f2`
cat << EOF >> $OUT
Google,gmail.com,com,USA,North America,$GOO1
Google,googlemail.com,com,Germany,Europe,$GOO2,Originally used in UK/Germany due to trademark dispute
EOF

#
#TLD/Country/Region Data
#Yes, I know the regions are pretty crappy and require cleanup
#I should have paid more attention in Civics class
DATA=",ac,Ascension Island,Oceania,
,ad,Andorra,Europe,
,ae,United Arab Emirates,Middle East,
,af,Afghanistan,Asia,
,ag,Antigua and Barbuda,Caribbean,
,ai,Anguilla,North America,
,al,Albania,Europe,
,am,Armenia,Asia,
,ao,Angola,Africa,
,aq,Antarctica,Antarctica,
,ar,Argentina,South America,
,as,American Samoa,Oceania,
,at,Austria,Europe,
,au,Australia,Oceania,
,aw,Aruba,Caribbean,
,ax,Ã…land,Europe,
,az,Azerbaijan,Asia,
,ba,Bosnia and Herzegovina,Europe,
,bb,Barbados,Caribbean,
,bd,Bangladesh,Asia,
,be,Belgium,Europe,
,bf,Burkina Faso,Africa,
,bg,Bulgaria,Europe,
,bh,Bahrain,Middle East,
,bi,Burundi,Africa,
,bj,Benin,Africa,
,bm,Bermuda,Caribbean,
,bn,Brunei,Asia,
,bo,Bolivia,South America,
,br,Brazil,South America,
,bs,Bahamas,Caribbean,
,bt,Bhutan,Asia,
,bv,Bouvet Island,Other,
,bw,Botswana,Africa,
,by,Belarus,Europe,
,bz,Belize,North America,
,ca,Canada,North America,
,cc,Cocos (Keeling) Islands,Oceania,
,cd,Democratic Republic of the Congo,Africa,
,cf,Central African Republic,Africa,
,cg,Republic of the Congo,Africa,
,ch,Switzerland,Europe,
,ci,Cote d'Ivoire,Africa,
,ck,Cook Islands,Oceania,
,cl,Chile,South America,
,cm,Cameroon,Africa,
,cn,People's Republic of China,Asia,
,co,Colombia,South America,
,com,USA,North America,
,cr,Costa Rica,North America,
,cs,Czechoslovakia,Europe,
,cu,Cuba,Caribbean,
,cv,Cape Verde,Africa,
,cw,Curacao,Caribbean,
,cx,Christmas Island,Oceania,
,cy,Cyprus,Europe,
,cz,Czech Republic,Europe,
,de,Germany,Europe,
,dj,Djibouti,Africa,
,dk,Denmark,Europe,
,dm,Dominica,Caribbean,
,do,Dominican Republic,Caribbean,
,dz,Algeria,Africa,
,ec,Ecuador,South America,
,ee,Estonia,Europe,
,eg,Egypt,Africa,
,eh,Western Sahara,Africa,
,er,Eritrea,Africa,
,es,Spain,Europe,
,et,Ethiopia,Africa,
,eu,European Union,Europe,
,fi,Finland,Europe,
,fj,Fiji,Oceania,
,fk,Falkland Islands,South America,
,fm,Federated States of Micronesia,Oceania,
,fo,Faroe Islands,Europe,
,fr,France,Europe,
,ga,Gabon,Africa,
,gb,United Kingdom,Europe,
,gd,Grenada,Caribbean,
,ge,Georgia,Europe,
,gf,French Guiana,South America,
,gg,Guernsey,Europe,
,gh,Ghana,Africa,
,gi,Gibraltar,Europe,
,gl,Greenland,Europe,
,gm,The Gambia,Africa,
,gn,Guinea,Africa,
,gp,Guadeloupe,North America,
,gq,Equatorial Guinea,Africa,
,gr,Greece,Europe,
,gs,South Georgia and the South Sandwich Islands,Oceania,
,gt,Guatemala,North America,
,gu,Guam,Oceania,
,gw,Guinea-Bissau,Africa,
,gy,Guyana,South America,
,hk,Hong Kong,Asia,
,hm,Heard Island and McDonald Islands,Oceania,
,hn,Honduras,North America,
,hr,Croatia,Europe,
,ht,Haiti,North America,
,hu,Hungary,Europe,
,id,Indonesia,Asia,
,ie,Ireland,Europe,
,il,Israel,Middle East,
,im,Isle of Man,Europe,
,in,India,Asia,
,io,British Indian Ocean Territory,Oceania,
,iq,Iraq,Middle East,
,ir,Iran,Middle East,
,is,Iceland,Europe,
,it,Italy,Europe,
,je,Jersey,Europe,
,jm,Jamaica,Caribbean,
,jo,Jordan,Middle East,
,jp,Japan,Asia,
,ke,Kenya,Africa,
,kg,Kyrgyzstan,Asia,
,kh,Cambodia,Asia,
,ki,Kiribati,Oceania,
,km,Comoros,Africa,
,kn,Saint Kitts and Nevis,Caribbean,
,kp,Democratic People's Republic of Korea,Asia,
,kr,Republic of Korea,Asia,
,kw,Kuwait,Middle East,
,ky,Cayman Islands,Caribbean,
,kz,Kazakhstan,Asia,
,la,Laos,Asia,
,lb,Lebanon,Middle East,
,lc,Saint Lucia,Caribbean,
,li,Liechtenstein,Europe,
,lk,Sri Lanka,Asia,
,lr,Liberia,Africa,
,ls,Lesotho,Africa,
,lt,Lithuania,Europe,
,lu,Luxembourg,Europe,
,lv,Latvia,Europe,
,ly,Libya,Africa,
,ma,Morocco,Africa,
,mc,Monaco,Europe,
,md,Moldova,Europe,
,me,Montenegro,Europe,
,mg,Madagascar,Africa,
,mh,Marshall Islands,Oceania,
,mk,Macedonia,Europe,
,ml,Mali,Africa,
,mm,Myanmar,Asia,
,mn,Mongolia,Asia,
,mo,Macau,Asia,
,mp,Northern Mariana Islands,Asia,
,mq,Martinique,North America,
,mr,Mauritania,Africa,
,ms,Montserrat,North America,
,mt,Malta,Europe,
,mu,Mauritius,Africa,
,mv,Maldives,Asia,
,mw,Malawi,Africa,
,mx,Mexico,North America,
,my,Malaysia,Asia,
,mz,Mozambique,Africa,
,na,Namibia,Africa
,nc,New Caledonia,Other,
,ne,Niger,Africa,
,net,USA,North America,
,nf,Norfolk Island,Other,
,ng,Nigeria,Africa,
,ni,Nicaragua,North America,
,nl,Netherlands,Europe,
,no,Norway,Europe,
,np,Nepal,Asia,
,nr,Nauru,Other,
,nu,Niue,Oceania,
,nz,New Zealand,Oceania,
,om,Oman,Middle East,
,org,USA,North America,
,pa,Panama,North America,
,pe,Peru,South America,
,pf,French Polynesia,Other,
,pg,Papua New Guinea,Asia,
,ph,Philippines,Asia,
,pk,Pakistan,Asia,
,pl,Poland,Europe,
,pm,Saint-Pierre and Miquelon,North America,
,pn,Pitcairn Islands,Oceania,
,pr,Puerto Rico,North America,
,ps,State of Palestine,Middle East,
,pt,Portugal,Europe,
,pw,Palau,Oceania,
,py,Paraguay,South America,
,qa,Qatar,Middle East,
,re,Reunion,Indian Ocean,
,ro,Romania,Europe,
,rs,Serbia,Europe,
,ru,Russia,Asia,
,rw,Rwanda,Africa,
,sa,Saudi Arabia,Middle East,
,sb,Solomon Islands,Oceania,
,sc,Seychelles,Africa,
,sd,Sudan,Africa,
,se,Sweden,Europe,
,sg,Singapore,Asia,
,sh,Saint Helena,Oceania,
,si,Slovenia,Europe,
,sk,Slovakia,Europe,
,sl,Sierra Leone,Africa,
,sm,San Marino,Europe,
,sn,Senegal,Africa,
,so,Somalia,Africa,
,sr,Suriname,South America,
,ss,South Sudan,Africa,
,st,Sao Tome and Principe,Africa,
,su,Soviet Union,Europe,
,sv,El Salvador,North America,
,sx,Sint Maarten,Caribbean,
,sy,Syria,Middle East,
,sz,Swaziland,Africa,
,tc,Turks and Caicos Islands,Caribbean,
,td,Chad,Africa,
,tf,French Southern and Antarctic Lands,Other,
,tg,Togo,Africa,
,th,Thailand,Asia,
,tj,Tajikistan,Asia,
,tk,Tokelau,Other,
,tl,East Timor,Other,
,tm,Turkmenistan,Asia,
,tn,Tunisia,Africa,
,to,Tonga,Oceania,
,tp,East Timor,Other,
,tr,Turkey,Asia,
,tt,Trinidad and Tobago,Caribbean,
,tv,Tuvalu,Oceania,
,tw,Taiwan,Asia,
,tz,Tanzania,Africa,
,ua,Ukraine,Asia,
,ug,Uganda,Africa,
,uk,United Kingdom,Europe,
,us,USA,North America,
,uy,Uruguay,South America,
,uz,Uzbekistan,Asia,
,va,Vatican City,Europe,
,vc,Saint Vincent and the Grenadines,Caribbean,
,ve,Venezuela,South America,
,vg,British Virgin Islands,Caribbean,
,vi,United States Virgin Islands,Caribbean,
,vn,Vietnam,Asia,
,vu,Vanuatu,Oceania,
,wf,Wallis and Futuna,Other,
,ws,Samoa,Oceania,
,ye,Yemen,Middle East,
,yt,Mayotte,Other,
,za,South Africa,Africa,
,zm,Zambia,Africa,
,zw,Zimbabwe,Africa,"

#
#List of possible Yahoo/Microsoft domain keywords
DOMPARTSL=",Yahoo,aim,
,Yahoo,aol,
,Yahoo,autoblog,
,Yahoo,bellatlantic,
,Yahoo,citlink,
,Yahoo,compuserve,
,Yahoo,cs,
,Yahoo,frontier,
,Yahoo,frontiernet,
,Yahoo,games,
,Yahoo,goowy,
,Yahoo,gte,
,Yahoo,huffingtonpost,
,Yahoo,love,
,Yahoo,netscape,
,Yahoo,oath,
,Yahoo,rocketmail,
,Yahoo,rogers,
,Yahoo,sky,
,Yahoo,teamaol,
,Yahoo,verizon,
,Yahoo,verizonmedia,
,Yahoo,wild4music,
,Yahoo,wmconnect,
,Yahoo,wow,
,Yahoo,xtra,
,Yahoo,y7mail,
,Yahoo,yahoo,
,Yahoo,yahooinc,
,Yahoo,yahoogroups,
,Yahoo,yahoogrupper,
,Yahoo,yahoogruppi,
,Yahoo,yahooxtra,
,Yahoo,ybb,
,Yahoo,ygm,
,Yahoo,ymail,
,Microsoft,msn,
,Microsoft,hotmail,
,Microsoft,live,
,Microsoft,outlook,
,Microsoft,webtv,"

function log_it {

#
#We've got a match, tell the user and log the output
echo "$DOMAIN is used by $ISP in $COUNTRYL ($MX)"
echo "$ISP,$DOMAIN,$COUNTRY,$COUNTRYL,$REGION,$MX,$1" >> $OUT

}

function check_mx {

#
#Lookup the MX record for the proposed domain
#To see if it matches an Yahoo or Microsoft MX
DSERVER=`pick_server`
HOST=`host -t mx "$DOMAIN" $DSERVER 2>&1`
MX=`echo "$HOST" | grep "is handled by" | grep -v "invalid" | sort | uniq | tail -1 | cut -d" " -f7`
ERR=`echo "$HOST" | grep -ai "[A-Z]" | tail -1`

#
#Something went wrong with the DNS check
if echo "$MX" | grep -qai "[A-Z]"
then
: do nothing
else
echo "$DOMAIN: No DNS found for $ISP ($ERR)"
return
fi

#
#No MX record found for a given domain
#In theory allowed per RFC. In practice these ISPs always have an MX record.
if echo "$HOST" | grep -qai " has no "
then
echo "$DOMAIN: No MX record found for $ISP ($ERR)"
return
fi

#
#At least one funky domain has a "nomail.yahoo.com" MX record
if echo "$HOST" | grep -ai "is handled by" | grep -qai "nomail.yahoo.com"
then
echo "$DOMAIN: Has nomail MX record for $ISP ($MX)"
return
fi

# 
#Some domains have "localhost" in the MX record, don't want mail
if echo "$HOST" | grep -ai "is handled by" | grep -qai "localhost"
then
echo "$DOMAIN: Has localhost MX record for $ISP ($MX)"
return
fi

#
#Pull the 2-letter country code from the TLD
if echo "$TLD" | cut -d"." -f2 | grep -qai "[A-Z]"
then
COUNTRY=`echo "$TLD" | cut -d"." -f2`
else
COUNTRY="$TLD"
fi

#
#Let's use the country code to find the country name and region
COUNTRYL=`echo "$DATA" | grep ",$COUNTRY," | head -1 | cut -d"," -f3`
REGION=`echo "$DATA" | grep ",$COUNTRY," | head -1 | cut -d"," -f4`

#
#Two different sets of tests, one for Microsoft, one for Yahoo
#At first I combined these, but then you find stuff like AIM.org is hosted by Microsoft
#So best to keep them seperate
if echo "$ISP" | grep -qai Microsoft
then

#
#Calling any *.outlook.com MX a match, even if it's not OLC
#These 4 Microsoft domains appear to be ISP/OLC-like domains but don't use "OLC" MX records:
#msn.net.au, msn.co.nz, live.co.hu, outlook.co.jp
if echo "$HOST" | grep "is handled" | cut -d" " -f7 | grep -qai ".outlook.com.$"
then
log_it
return
fi

#This catches
#msn.cn, live.lk, live.mk, outlook.com.cn
#But I don't think they count as sancioned OLC domains
#Leaving this for now
if echo "$HOST" | grep "is handled" | cut -d" " -f7 | grep -qai ".hotmail.com.$"
then
log_it "Non-standard hotmail.com MX but hostname suggests OLC"
return
fi

else

#
#Almost all Yahoo domains have an MX record ending in yahoodns.net
if echo "$HOST" | grep -ai "is handled by" | grep -qai ".yahoodns.net.$"
then
log_it
return
fi

#
#Except Yahoo Japan
if echo "$HOST" | grep -ai "is handled by" | grep -qai ".mail.yahoo.co.jp.$"
then
log_it "Yahoo Japan"
return
fi

#
#This catches yahoo.co.hu, yahoo.rs, yahoo.si, yahooxtra.co.nz
#Are these Yahoo consumer domains? Unsure, so best to include them.
if echo "$HOST" | grep -ai "is handled by" | grep -qai ".yahoo.com.$"
then
log_it "Consumer domain? Non-standard yahoo.com MX"
return
fi

fi

#
#If we got this far, we found no matching Yahoo or Microsoft MX record in DNS
echo "$DOMAIN: No matching MX found for $ISP ($MX)"

}

#
#This is the big loop that does all the magic
#It takes every DOMPART and combines it with every possible TLD
#Then looks for an MX record for that domain
echo "$DOMPARTSL" | \
sort | uniq | (
while read DOMPARTS
do

DOMPART=`echo "$DOMPARTS" | cut -d"," -f3`
ISP=`echo "$DOMPARTS" | cut -d"," -f2`

#Extract a list of TLDs from the DATA array
echo "$DATA" | \
cut -d"," -f2 | \
tr "[A-Z]" "[a-z]" | \
sort | uniq | (
while read TLD
do

#
#A few different variations are tried
#for stuff like .co.jp .co.uk .ne.jp etc.
#Would be faster to limit this to certain TLDs,
#But I'm lazy; let's let the script figure it out from DNS
LOOP="$DOMPART.$TLD
$DOMPART.co.$TLD
$DOMPART.com.$TLD
$DOMPART.ne.$TLD
$DOMPART.net.$TLD"

echo "$LOOP" | (
while read DOMAIN
do

check_mx

done
)

done
)

done
)

#
#Manual cleanup: #Label y7mail as .au
#and sky as .uk
cat $OUT | \
sed "s/,sky.com,com,USA,North America,/,sky.com,com,United Kingdom,Europe,/g" | \
sed "s/,y7mail.com,com,USA,North America,/,y7mail.com,com,Australia,Oceania,/g" > $OUT.2
mv $OUT.2 $OUT

ENDDAT=`date "+%s"`
ENDDAT2=`date`
ELAPSED=`echo "($ENDDAT-$STARTDAT)/60"|bc`

cat << EOF
Start:   $STARTDAT2
End:     $ENDDAT2
Runtime: $ELAPSED minutes
EOF

#
#That was fun! Don't you love reading comments in shell scripts?