diff --git a/common/common.go b/common/common.go index 2dccefe0..213991bf 100644 --- a/common/common.go +++ b/common/common.go @@ -56,7 +56,7 @@ var ErrNoRows = sql.ErrNoRows // TODO: Make this more customisable var SpammyDomainBits = []string{"porn", "sex", "lesbian", "acup", "nude", "milf", "tits", "vape", "busty", "kink", "lingerie", "strapon", "problog", "fet", "xblog", "blogin", "blognetwork", "relayblog"} -var Chrome, Firefox, Semrush int // ! Temporary Hack for http push and stopping semrush from wasting resources +var Chrome, Firefox, Semrush, Ahrefs int // ! Temporary Hack for http push and stopping semrush from wasting resources type StringList []string diff --git a/gen_router.go b/gen_router.go index b4132359..a80cade4 100644 --- a/gen_router.go +++ b/gen_router.go @@ -694,6 +694,7 @@ var markToAgent = map[string]string{ "360Spider": "haosou", "bingbot": "bing", "BingPreview": "bing", + "msnbot": "bing", "Slurp": "slurp", "Exabot": "exabot", "MojeekBot": "mojeek", @@ -744,6 +745,7 @@ var markToID = map[string]int{ "360Spider": 22, "bingbot": 14, "BingPreview": 14, + "msnbot": 14, "Slurp": 15, "Exabot": 16, "MojeekBot": 17, @@ -790,6 +792,7 @@ func init() { c.Chrome = agentMapEnum["chrome"] c.Firefox = agentMapEnum["firefox"] c.Semrush = agentMapEnum["semrush"] + c.Ahrefs = agentMapEnum["ahrefs"] } type WriterIntercept struct { diff --git a/router_gen/main.go b/router_gen/main.go index 1e9d0d47..45cbc623 100644 --- a/router_gen/main.go +++ b/router_gen/main.go @@ -310,6 +310,7 @@ func main() { "360Spider", "bingbot", "BingPreview", + "msnbot", "Slurp", "Exabot", "MojeekBot", @@ -363,6 +364,7 @@ func main() { "360Spider": "haosou", "bingbot": "bing", "BingPreview": "bing", + "msnbot":"bing", "Slurp": "slurp", "Exabot": "exabot", "MojeekBot": "mojeek", @@ -471,6 +473,7 @@ func init() { c.Chrome = agentMapEnum["chrome"] c.Firefox = agentMapEnum["firefox"] c.Semrush = agentMapEnum["semrush"] + c.Ahrefs = agentMapEnum["ahrefs"] } type WriterIntercept struct { diff --git a/routes/common.go b/routes/common.go index dbb16cfa..3abb56be 100644 --- a/routes/common.go +++ b/routes/common.go @@ -119,8 +119,8 @@ func FootHeaders(w http.ResponseWriter, h *c.Header) { func renderTemplate3(tmplName, hookName string, w http.ResponseWriter, r *http.Request, h *c.Header, pi interface{}) error { s := h.Stylesheets h.Stylesheets = nil - jsEnable := h.CurrentUser.LastAgent != c.Semrush - if r.FormValue("i") != "1" && h.CurrentUser.LastAgent != c.Semrush { + simpleBot := h.CurrentUser.LastAgent == c.Semrush || h.CurrentUser.LastAgent == c.Ahrefs + if r.FormValue("i") != "1" && !simpleBot { c.PrepResources(h.CurrentUser, h, h.Theme) for _, ss := range s { h.Stylesheets = append(h.Stylesheets, ss) @@ -132,14 +132,14 @@ func renderTemplate3(tmplName, hookName string, w http.ResponseWriter, r *http.R h.CurrentUser.LastAgent = 0 } - if h.CurrentUser.Loggedin || h.CurrentUser.LastAgent == c.Semrush { + if h.CurrentUser.Loggedin || simpleBot { h.MetaDesc = "" h.OGDesc = "" } else if h.MetaDesc != "" && h.OGDesc == "" { h.OGDesc = h.MetaDesc } - if jsEnable { + if !simpleBot { FootHeaders(w, h) } if h.Zone != "error" {