webfs: remove automatic Referer header

automatic Referer headers were found to break downloads from
sourceforge.net, causing html pages to appear instead of tarballs.

if a website does need them, they can be added as needed via hget's -r
flag, or added to mothra.

reverts commits 67f536d20329 and 6d999c39a9f0
This commit is contained in:
mischief 2018-09-18 13:14:00 -07:00
parent 6b9cc409de
commit 026c679f49

View file

@ -407,33 +407,6 @@ fsopen(Req *r)
if(!lookkey(cl->hdr, "Accept"))
cl->hdr = addkey(cl->hdr, "Accept", "*/*");
if(!lookkey(cl->hdr, "Referer")){
char *r;
Url *u;
/*
* Referer header is often required on broken
* websites even if the spec makes them optional,
* so we make one up.
*/
if(u = url("/", cl->url)){
if(r = u->host){
u->host = smprint("%N", r);
free(r);
}
/* do not send credentials */
free(u->user); u->user = nil;
free(u->pass); u->pass = nil;
if(r = smprint("%U", u)){
cl->hdr = addkey(cl->hdr, "Referer", r);
free(r);
}
freeurl(u);
}
}
if(!lookkey(cl->hdr, "Connection"))
cl->hdr = addkey(cl->hdr, "Connection", "keep-alive");