vote up 0 vote down star

I need an online or offline app that can take a URL, vary the input value for one parameter according to rules set by the user, and then download each varied url it gets. For example, if I give it www.site.com/page.php?number=1. Then it should replace the '1' with the sequence of numbers I give it, and download
www.site.com/page.php?number=1
www.site.com/page.php?number=2
www.site.com/page.php?number=3
...

Something like a webspider. Also, if the program could automatically show only the elements that are different in every page, or allow the user to parse the data and make a table only of the changes, that would be great. For example, if the first url returns "My name is A", the second returns "My name is B", the third replies "My name is C", I want the program to be able to output:
1 : A
2 : B
3 : C
...

Is this possible? Is there anything out there like this?

flag

1 Answer

vote up 2 vote down

If you're passing parameters with a question mark like you have shown then that is a GET, and not a POST.

In general terms you're going to want to write a script to output all the urls to a text file, which should be a simple for/while loop. You can then use wget to download all of these (wget -i links.txt). And finally you'll want to write a script with a regex to pull the relevant data. That is of course unless you can find software to do it all for you.

To get the list of urls, if they are all GET requests with incremental numbers passed, visit the first one and then use the following javascript bookmarklet:

javascript:(function(){%20function%20selectColor(i)%20{%20return%20["#fdc",%20"#cdf",%20"#bfd",%20"#dbf",%20"#fbd"]%20[i%5];%20}%20var%20u=location.href,%20ul=u.length;%20var%20tparts=[""],%20zparts=[],%20nz=0;%20function%20isDigit(c)%20{%20return%20("0"%20<=%20c%20&&%20c%20<=%20"9");%20}%20for%20(i=0;%20i<ul;%20)%20{%20for%20(;%20i<ul%20&&%20!isDigit(u.charAt(i));%20++i)%20tparts[nz]%20+=%20u.charAt(i);%20if(i<ul)%20{%20zparts[nz]="";%20for%20(;%20i<ul%20&&%20isDigit(u.charAt(i));%20++i)%20zparts[nz]%20+=%20u.charAt(i);%20tparts[nz+1]="";%20++nz;%20}%20}%20if(!nz)%20{%20alert("No%20numbers%20in%20URL.");%20return;%20}%20D=window.open().document;%20D.write();%20D.close();%20function%20a(n)%20{%20A(D.body,n);%20}%20function%20A(p,n)%20{%20p.appendChild(n);%20}%20function%20E(q)%20{%20return%20D.createElement(q);%20}%20function%20cT(t)%20{%20return%20D.createTextNode(t)%20}%20function%20cBR()%20{%20return%20E("br");%20}%20function%20cS(t,ci)%20{%20var%20s=E("span");%20s.style.background=selectColor(ci);%20s.style.fontWeight="bold";%20A(s,%20cT(t));%20return%20s;%20}%20function%20cTB(v,oc)%20{%20var%20b=E("input");%20b.size=6;%20b.value=v;%20b.addEventListener("input",%20oc,%20false);%20return%20b;%20}%20function%20cCB(t,oc)%20{%20var%20L=E("label"),%20b=E("input");%20b.type="checkbox";%20b.checked=true;%20b.onchange=oc;%20A(L,b);%20A(L,cT(t));%20return%20L;%20}%20function%20cL(nz,tparts,zparts)%20{%20var%20L=E("a");%20var%20u="";%20for%20(var%20i=0;%20i<nz;%20++i)%20{%20A(L,cT(tparts[i]));%20A(L,cS(zparts[i],%20i));%20u%20+=%20tparts[i]+zparts[i];%20}%20A(L,cT(tparts[nz]));%20u%20+=%20tparts[nz];%20L.href=u;%20L.target="_blank";%20return%20L;%20}%20a(cT("Original%20URL:%20"));%20a(cBR());%20a(cL(nz,%20tparts,%20zparts));%20a(cBR());%20a(cBR());%20var%20fromBoxes=[],%20toBoxes=[],%20padChecks=[];%20for%20(i=0;%20i<nz;%20++i)%20{%20a(cT("Run%20"));%20a(cS(zparts[i],%20i));%20a(cT("%20from%20"));%20a(fromBoxes[i]=cTB(zparts[i],%20listURLs));%20a(cT("%20to%20"));%20a(toBoxes[i]=cTB(zparts[i],%20listURLs));%20a(cT("%20("));%20a(j=cCB("%20Pad%20with%20zeroes%20to%20maintain%20length",%20listURLs));%20padChecks[i]=j.childNodes[0];%20a(cT(")"));%20a(cBR());%20}%20a(cBR());%20resultDiv=E("div");%20a(resultDiv);%20listURLs();%20function%20listURLs()%20{%20while%20(resultDiv.childNodes.length)%20resultDiv.removeChild(resultDiv.childNodes[0]);%20var%20lows=[],%20highs=[];%20for%20(i=0;%20i<nz;%20++i)%20{%20lows[i]=parseInt(fromBoxes[i].value,%2010);%20highs[i]=parseInt(toBoxes[i].value,%2010);%20if(highs[i]-lows[i]%20>%20999)%20{%20A(resultDiv,%20cT("Too%20many"));%20return;%20}%20}%20urls=[];%20function%20cb(sta)%20{%20var%20newzparts=[];%20for%20(var%20i=0;%20i<nz;%20++i)%20{%20var%20z=""+sta[i];%20if(padChecks[i].checked)%20while%20(z.length%20<%20zparts[i].length)%20z="0"+z;%20newzparts[i]=z;%20}%20A(resultDiv,%20cL(nz,%20tparts,%20newzparts));%20A(resultDiv,%20cBR());%20}%20fors(nz,%20cb,%20lows,%20highs);%20}%20function%20fors%20(n,%20callback,%20lows,%20highs)%20{%20function%20fors_inner%20(states,%20v)%20{%20if(v%20>=%20n)%20callback(states);%20else%20for%20(states[v]=lows[v];%20states[v]%20<=%20highs[v];%20++(states[v]))%20fors_inner(states,%20v+1);%20}%20fors_inner%20([],%200);%20}%20})()

So now you're just left with having to write a script to find the text which differs between each file.

link|flag
Hey, thanks for that awesome bookmarklet. I'll wait for some time to see if anyone can post any software that does the regex part for me, otherwise I'll write the script myself. – Neil Jan 10 at 17:26

Your Answer

Get an OpenID
or
never shown

Not the answer you're looking for? Browse other questions tagged or ask your own question.