map msnbot to bing and avoid bytes for ahrefs too

common/common.go

@@ -56,7 +56,7 @@ var ErrNoRows = sql.ErrNoRows
 // TODO: Make this more customisable
 var SpammyDomainBits = []string{"porn", "sex", "lesbian", "acup", "nude", "milf", "tits", "vape", "busty", "kink", "lingerie", "strapon", "problog", "fet", "xblog", "blogin", "blognetwork", "relayblog"}
 
-var Chrome, Firefox, Semrush int // ! Temporary Hack for http push and stopping semrush from wasting resources
+var Chrome, Firefox, Semrush, Ahrefs int // ! Temporary Hack for http push and stopping semrush from wasting resources
 
 type StringList []string
 

gen_router.go

@@ -694,6 +694,7 @@ var markToAgent = map[string]string{
 	"360Spider": "haosou",
 	"bingbot": "bing",
 	"BingPreview": "bing",
+	"msnbot": "bing",
 	"Slurp": "slurp",
 	"Exabot": "exabot",
 	"MojeekBot": "mojeek",
@@ -744,6 +745,7 @@ var markToID = map[string]int{
 	"360Spider": 22,
 	"bingbot": 14,
 	"BingPreview": 14,
+	"msnbot": 14,
 	"Slurp": 15,
 	"Exabot": 16,
 	"MojeekBot": 17,
@@ -790,6 +792,7 @@ func init() {
 	c.Chrome = agentMapEnum["chrome"]
 	c.Firefox = agentMapEnum["firefox"]
 	c.Semrush = agentMapEnum["semrush"]
+	c.Ahrefs = agentMapEnum["ahrefs"]
 }
 
 type WriterIntercept struct {

router_gen/main.go

@@ -310,6 +310,7 @@ func main() {
 		"360Spider",
 		"bingbot",
 		"BingPreview",
+		"msnbot",
 		"Slurp",
 		"Exabot",
 		"MojeekBot",
@@ -363,6 +364,7 @@ func main() {
 		"360Spider":           "haosou",
 		"bingbot":             "bing",
 		"BingPreview":         "bing",
+		"msnbot":"bing",
 		"Slurp":               "slurp",
 		"Exabot":              "exabot",
 		"MojeekBot":           "mojeek",
@@ -471,6 +473,7 @@ func init() {
 	c.Chrome = agentMapEnum["chrome"]
 	c.Firefox = agentMapEnum["firefox"]
 	c.Semrush = agentMapEnum["semrush"]
+	c.Ahrefs = agentMapEnum["ahrefs"]
 }
 
 type WriterIntercept struct {

routes/common.go

@@ -119,8 +119,8 @@ func FootHeaders(w http.ResponseWriter, h *c.Header) {
 func renderTemplate3(tmplName, hookName string, w http.ResponseWriter, r *http.Request, h *c.Header, pi interface{}) error {
 	s := h.Stylesheets
 	h.Stylesheets = nil
-	jsEnable := h.CurrentUser.LastAgent != c.Semrush
-	if r.FormValue("i") != "1" && h.CurrentUser.LastAgent != c.Semrush {
+	simpleBot := h.CurrentUser.LastAgent == c.Semrush || h.CurrentUser.LastAgent == c.Ahrefs
+	if r.FormValue("i") != "1" && !simpleBot {
 		c.PrepResources(h.CurrentUser, h, h.Theme)
 		for _, ss := range s {
 			h.Stylesheets = append(h.Stylesheets, ss)
@@ -132,14 +132,14 @@ func renderTemplate3(tmplName, hookName string, w http.ResponseWriter, r *http.R
 		h.CurrentUser.LastAgent = 0
 	}
 
-	if h.CurrentUser.Loggedin || h.CurrentUser.LastAgent == c.Semrush {
+	if h.CurrentUser.Loggedin || simpleBot {
 		h.MetaDesc = ""
 		h.OGDesc = ""
 	} else if h.MetaDesc != "" && h.OGDesc == "" {
 		h.OGDesc = h.MetaDesc
 	}
 
-	if jsEnable {
+	if !simpleBot {
 		FootHeaders(w, h)
 	}
 	if h.Zone != "error" {