map msnbot to bing and avoid bytes for ahrefs too

This commit is contained in:
Azareal 2020-03-19 07:21:06 +10:00
parent 7d25937352
commit 6a7b8fd3ee
4 changed files with 11 additions and 5 deletions

View File

@ -56,7 +56,7 @@ var ErrNoRows = sql.ErrNoRows
// TODO: Make this more customisable // TODO: Make this more customisable
var SpammyDomainBits = []string{"porn", "sex", "lesbian", "acup", "nude", "milf", "tits", "vape", "busty", "kink", "lingerie", "strapon", "problog", "fet", "xblog", "blogin", "blognetwork", "relayblog"} var SpammyDomainBits = []string{"porn", "sex", "lesbian", "acup", "nude", "milf", "tits", "vape", "busty", "kink", "lingerie", "strapon", "problog", "fet", "xblog", "blogin", "blognetwork", "relayblog"}
var Chrome, Firefox, Semrush int // ! Temporary Hack for http push and stopping semrush from wasting resources var Chrome, Firefox, Semrush, Ahrefs int // ! Temporary Hack for http push and stopping semrush from wasting resources
type StringList []string type StringList []string

View File

@ -694,6 +694,7 @@ var markToAgent = map[string]string{
"360Spider": "haosou", "360Spider": "haosou",
"bingbot": "bing", "bingbot": "bing",
"BingPreview": "bing", "BingPreview": "bing",
"msnbot": "bing",
"Slurp": "slurp", "Slurp": "slurp",
"Exabot": "exabot", "Exabot": "exabot",
"MojeekBot": "mojeek", "MojeekBot": "mojeek",
@ -744,6 +745,7 @@ var markToID = map[string]int{
"360Spider": 22, "360Spider": 22,
"bingbot": 14, "bingbot": 14,
"BingPreview": 14, "BingPreview": 14,
"msnbot": 14,
"Slurp": 15, "Slurp": 15,
"Exabot": 16, "Exabot": 16,
"MojeekBot": 17, "MojeekBot": 17,
@ -790,6 +792,7 @@ func init() {
c.Chrome = agentMapEnum["chrome"] c.Chrome = agentMapEnum["chrome"]
c.Firefox = agentMapEnum["firefox"] c.Firefox = agentMapEnum["firefox"]
c.Semrush = agentMapEnum["semrush"] c.Semrush = agentMapEnum["semrush"]
c.Ahrefs = agentMapEnum["ahrefs"]
} }
type WriterIntercept struct { type WriterIntercept struct {

View File

@ -310,6 +310,7 @@ func main() {
"360Spider", "360Spider",
"bingbot", "bingbot",
"BingPreview", "BingPreview",
"msnbot",
"Slurp", "Slurp",
"Exabot", "Exabot",
"MojeekBot", "MojeekBot",
@ -363,6 +364,7 @@ func main() {
"360Spider": "haosou", "360Spider": "haosou",
"bingbot": "bing", "bingbot": "bing",
"BingPreview": "bing", "BingPreview": "bing",
"msnbot":"bing",
"Slurp": "slurp", "Slurp": "slurp",
"Exabot": "exabot", "Exabot": "exabot",
"MojeekBot": "mojeek", "MojeekBot": "mojeek",
@ -471,6 +473,7 @@ func init() {
c.Chrome = agentMapEnum["chrome"] c.Chrome = agentMapEnum["chrome"]
c.Firefox = agentMapEnum["firefox"] c.Firefox = agentMapEnum["firefox"]
c.Semrush = agentMapEnum["semrush"] c.Semrush = agentMapEnum["semrush"]
c.Ahrefs = agentMapEnum["ahrefs"]
} }
type WriterIntercept struct { type WriterIntercept struct {

View File

@ -119,8 +119,8 @@ func FootHeaders(w http.ResponseWriter, h *c.Header) {
func renderTemplate3(tmplName, hookName string, w http.ResponseWriter, r *http.Request, h *c.Header, pi interface{}) error { func renderTemplate3(tmplName, hookName string, w http.ResponseWriter, r *http.Request, h *c.Header, pi interface{}) error {
s := h.Stylesheets s := h.Stylesheets
h.Stylesheets = nil h.Stylesheets = nil
jsEnable := h.CurrentUser.LastAgent != c.Semrush simpleBot := h.CurrentUser.LastAgent == c.Semrush || h.CurrentUser.LastAgent == c.Ahrefs
if r.FormValue("i") != "1" && h.CurrentUser.LastAgent != c.Semrush { if r.FormValue("i") != "1" && !simpleBot {
c.PrepResources(h.CurrentUser, h, h.Theme) c.PrepResources(h.CurrentUser, h, h.Theme)
for _, ss := range s { for _, ss := range s {
h.Stylesheets = append(h.Stylesheets, ss) h.Stylesheets = append(h.Stylesheets, ss)
@ -132,14 +132,14 @@ func renderTemplate3(tmplName, hookName string, w http.ResponseWriter, r *http.R
h.CurrentUser.LastAgent = 0 h.CurrentUser.LastAgent = 0
} }
if h.CurrentUser.Loggedin || h.CurrentUser.LastAgent == c.Semrush { if h.CurrentUser.Loggedin || simpleBot {
h.MetaDesc = "" h.MetaDesc = ""
h.OGDesc = "" h.OGDesc = ""
} else if h.MetaDesc != "" && h.OGDesc == "" { } else if h.MetaDesc != "" && h.OGDesc == "" {
h.OGDesc = h.MetaDesc h.OGDesc = h.MetaDesc
} }
if jsEnable { if !simpleBot {
FootHeaders(w, h) FootHeaders(w, h)
} }
if h.Zone != "error" { if h.Zone != "error" {