map msnbot to bing and avoid bytes for ahrefs too
This commit is contained in:
parent
7d25937352
commit
6a7b8fd3ee
|
@ -56,7 +56,7 @@ var ErrNoRows = sql.ErrNoRows
|
||||||
// TODO: Make this more customisable
|
// TODO: Make this more customisable
|
||||||
var SpammyDomainBits = []string{"porn", "sex", "lesbian", "acup", "nude", "milf", "tits", "vape", "busty", "kink", "lingerie", "strapon", "problog", "fet", "xblog", "blogin", "blognetwork", "relayblog"}
|
var SpammyDomainBits = []string{"porn", "sex", "lesbian", "acup", "nude", "milf", "tits", "vape", "busty", "kink", "lingerie", "strapon", "problog", "fet", "xblog", "blogin", "blognetwork", "relayblog"}
|
||||||
|
|
||||||
var Chrome, Firefox, Semrush int // ! Temporary Hack for http push and stopping semrush from wasting resources
|
var Chrome, Firefox, Semrush, Ahrefs int // ! Temporary Hack for http push and stopping semrush from wasting resources
|
||||||
|
|
||||||
type StringList []string
|
type StringList []string
|
||||||
|
|
||||||
|
|
|
@ -694,6 +694,7 @@ var markToAgent = map[string]string{
|
||||||
"360Spider": "haosou",
|
"360Spider": "haosou",
|
||||||
"bingbot": "bing",
|
"bingbot": "bing",
|
||||||
"BingPreview": "bing",
|
"BingPreview": "bing",
|
||||||
|
"msnbot": "bing",
|
||||||
"Slurp": "slurp",
|
"Slurp": "slurp",
|
||||||
"Exabot": "exabot",
|
"Exabot": "exabot",
|
||||||
"MojeekBot": "mojeek",
|
"MojeekBot": "mojeek",
|
||||||
|
@ -744,6 +745,7 @@ var markToID = map[string]int{
|
||||||
"360Spider": 22,
|
"360Spider": 22,
|
||||||
"bingbot": 14,
|
"bingbot": 14,
|
||||||
"BingPreview": 14,
|
"BingPreview": 14,
|
||||||
|
"msnbot": 14,
|
||||||
"Slurp": 15,
|
"Slurp": 15,
|
||||||
"Exabot": 16,
|
"Exabot": 16,
|
||||||
"MojeekBot": 17,
|
"MojeekBot": 17,
|
||||||
|
@ -790,6 +792,7 @@ func init() {
|
||||||
c.Chrome = agentMapEnum["chrome"]
|
c.Chrome = agentMapEnum["chrome"]
|
||||||
c.Firefox = agentMapEnum["firefox"]
|
c.Firefox = agentMapEnum["firefox"]
|
||||||
c.Semrush = agentMapEnum["semrush"]
|
c.Semrush = agentMapEnum["semrush"]
|
||||||
|
c.Ahrefs = agentMapEnum["ahrefs"]
|
||||||
}
|
}
|
||||||
|
|
||||||
type WriterIntercept struct {
|
type WriterIntercept struct {
|
||||||
|
|
|
@ -310,6 +310,7 @@ func main() {
|
||||||
"360Spider",
|
"360Spider",
|
||||||
"bingbot",
|
"bingbot",
|
||||||
"BingPreview",
|
"BingPreview",
|
||||||
|
"msnbot",
|
||||||
"Slurp",
|
"Slurp",
|
||||||
"Exabot",
|
"Exabot",
|
||||||
"MojeekBot",
|
"MojeekBot",
|
||||||
|
@ -363,6 +364,7 @@ func main() {
|
||||||
"360Spider": "haosou",
|
"360Spider": "haosou",
|
||||||
"bingbot": "bing",
|
"bingbot": "bing",
|
||||||
"BingPreview": "bing",
|
"BingPreview": "bing",
|
||||||
|
"msnbot":"bing",
|
||||||
"Slurp": "slurp",
|
"Slurp": "slurp",
|
||||||
"Exabot": "exabot",
|
"Exabot": "exabot",
|
||||||
"MojeekBot": "mojeek",
|
"MojeekBot": "mojeek",
|
||||||
|
@ -471,6 +473,7 @@ func init() {
|
||||||
c.Chrome = agentMapEnum["chrome"]
|
c.Chrome = agentMapEnum["chrome"]
|
||||||
c.Firefox = agentMapEnum["firefox"]
|
c.Firefox = agentMapEnum["firefox"]
|
||||||
c.Semrush = agentMapEnum["semrush"]
|
c.Semrush = agentMapEnum["semrush"]
|
||||||
|
c.Ahrefs = agentMapEnum["ahrefs"]
|
||||||
}
|
}
|
||||||
|
|
||||||
type WriterIntercept struct {
|
type WriterIntercept struct {
|
||||||
|
|
|
@ -119,8 +119,8 @@ func FootHeaders(w http.ResponseWriter, h *c.Header) {
|
||||||
func renderTemplate3(tmplName, hookName string, w http.ResponseWriter, r *http.Request, h *c.Header, pi interface{}) error {
|
func renderTemplate3(tmplName, hookName string, w http.ResponseWriter, r *http.Request, h *c.Header, pi interface{}) error {
|
||||||
s := h.Stylesheets
|
s := h.Stylesheets
|
||||||
h.Stylesheets = nil
|
h.Stylesheets = nil
|
||||||
jsEnable := h.CurrentUser.LastAgent != c.Semrush
|
simpleBot := h.CurrentUser.LastAgent == c.Semrush || h.CurrentUser.LastAgent == c.Ahrefs
|
||||||
if r.FormValue("i") != "1" && h.CurrentUser.LastAgent != c.Semrush {
|
if r.FormValue("i") != "1" && !simpleBot {
|
||||||
c.PrepResources(h.CurrentUser, h, h.Theme)
|
c.PrepResources(h.CurrentUser, h, h.Theme)
|
||||||
for _, ss := range s {
|
for _, ss := range s {
|
||||||
h.Stylesheets = append(h.Stylesheets, ss)
|
h.Stylesheets = append(h.Stylesheets, ss)
|
||||||
|
@ -132,14 +132,14 @@ func renderTemplate3(tmplName, hookName string, w http.ResponseWriter, r *http.R
|
||||||
h.CurrentUser.LastAgent = 0
|
h.CurrentUser.LastAgent = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
if h.CurrentUser.Loggedin || h.CurrentUser.LastAgent == c.Semrush {
|
if h.CurrentUser.Loggedin || simpleBot {
|
||||||
h.MetaDesc = ""
|
h.MetaDesc = ""
|
||||||
h.OGDesc = ""
|
h.OGDesc = ""
|
||||||
} else if h.MetaDesc != "" && h.OGDesc == "" {
|
} else if h.MetaDesc != "" && h.OGDesc == "" {
|
||||||
h.OGDesc = h.MetaDesc
|
h.OGDesc = h.MetaDesc
|
||||||
}
|
}
|
||||||
|
|
||||||
if jsEnable {
|
if !simpleBot {
|
||||||
FootHeaders(w, h)
|
FootHeaders(w, h)
|
||||||
}
|
}
|
||||||
if h.Zone != "error" {
|
if h.Zone != "error" {
|
||||||
|
|
Loading…
Reference in New Issue