Jump to content

Sky Slate Blueberry Blackcurrant Watermelon Strawberry Orange Banana Apple Emerald Chocolate
Photo

[Func] HTTPRequest: for Web APIs [AHK-B + AHK-L+Unicode+x64]


  • Please log in to reply
380 replies to this topic
VxE
  • Moderators
  • 3622 posts
  • Last active: Dec 24 2015 02:21 AM
  • Joined: 07 Oct 2006
Well, I found the solution, if not the problem. The referrer does indeed fail when using AHK-L (unicode). However, when I explicitly add the 'referer' header (see line 225), it works. I'm inclined to think there is a bug in HttpOpenRequestW, perhaps that function spells 'referer' correctly.

The updated version (10-14-2011) is available for download in the OP.

DerBen
  • Members
  • 7 posts
  • Last active: Oct 16 2011 01:20 AM
  • Joined: 27 Jun 2010

Well, I found the solution, if not the problem. The referrer does indeed fail when using AHK-L (unicode). However, when I explicitly add the 'referer' header (see line 225), it works.


I was just looking at that myself wondering why it was commented out... Thanks much! Seems to work as it should now. Big thanks on the replies too.

nimda
  • Members
  • 4368 posts
  • Last active: Aug 09 2015 02:36 AM
  • Joined: 26 Dec 2010
[VxE] I have a request. I would like to add Unicode support, therefore I'm transitioning to HTTPRequest from HTTPQuery. How would I replicate the following code (from Zizorz)?
image := newfilename
   FileGetSize,size,%image%
   SplitPath,image,OFN
   FileRead,img,%image%
   VarSetCapacity(placeholder,size,32)
   boundary := makeProperBoundary()
   post:="--" boundary "`ncontent-disposition: form-data; name=""MAX_FILE_SIZE""`n`n"
      . "1048576`n--" boundary "`ncontent-disposition: form-data; name=""xml""`n`nyes`n--"
      . boundary "`ncontent-disposition: form-data; name=""fileupload""; filename="""
      . ofn """`nContent-type: " MimeType(img) "`nContent-Transfer-Encoding: binary`n`n" 
      . placeholder "`n--" boundary "--"
   headers:="Content-type: multipart/form-data, boundary=" boundary "`nContent-Length: " strlen(post)
   DllCall("RtlMoveMemory","uInt",(offset:=&post+strlen(post)-strlen(Boundary)-size-5)
         ,"uInt",&img,"uInt",size)
   size := httpQuery(result:="","http://www.imageshack.us/index.php",post,headers)
   VarSetCapacity(result,-1)

makeProperBoundary(){
   Loop,26
      n .= chr(64+a_index)
   n .= "0123456789"
   Loop,% StrLen(A_Now) {
      Random,rnd,1,% StrLen(n)
      Random,UL,0,1
      b .= RegExReplace(SubStr(n,rnd,1),".$","$" (round(UL)? "U":"L") "0")
   }
   Return b
}

MimeType(ByRef Binary) {
   MimeTypes:="424d image/bmp|4749463 image/gif|ffd8ffe image/jpeg|89504e4 image/png|4657530"
          . " application/x-shockwave-flash|49492a0 image/tiff"
   @:="0123456789abcdef"
   Loop,8
      hex .= substr(@,(*(a:=&Binary-1+a_index)>>4)+1,1) substr(@,((*a)&15)+1,1)
   Loop,Parse,MimeTypes,|
      if ((substr(hex,1,strlen(n:=RegExReplace(A_Loopfield,"\s.*"))))=n)
         Mime := RegExReplace(A_LoopField,".*?\s")
   Return (Mime!="") ? Mime : "application/octet-stream"
}
I honestly have no idea what is happening there :(
although in the end, what you wind up with is the image has been uploaded to imageshack and the link is somewhere in result

VxE
  • Moderators
  • 3622 posts
  • Last active: Dec 24 2015 02:21 AM
  • Joined: 07 Oct 2006
Unfortuantely, multipart/form-data POST does not work with HTTPRequest's codepage conversion feature, so a workaround is needed for unicode versions of AHK.

For ANSI, simply replacing the function call is sufficient:
; ANSI
   FileGetSize,size,%image%
   FileRead,img,*%image%
   VarSetCapacity(placeholder,size,32)
   boundary := makeProperBoundary()
   post:="--" boundary "`ncontent-disposition: form-data; name=""MAX_FILE_SIZE""`n`n"
      . "1048576`n--" boundary "`ncontent-disposition: form-data; name=""xml""`n`nyes`n--"
      . boundary "`ncontent-disposition: form-data; name=""fileupload""; filename="""
      . RegexReplace( "\" image, ".*\\" ) """`nContent-type: " MimeType(img)
      . "`nContent-Transfer-Encoding: binary`n`n" placeholder "`n--" boundary "--"
   headers:="Content-type: multipart/form-data, boundary=" boundary "`nContent-Length: " strlen(post)
   DllCall("RtlMoveMemory","uInt",(offset:=&post+strlen(post)-strlen(Boundary)-size-5)
         ,"uInt",&img,"uInt",size)
;   size := httpQuery(result:="","http://www.imageshack.us/index.php",post,headers)
;   VarSetCapacity(result,-1)
	s := HTTPRequest( "http://www.imageshack.us/index.php", post, headers )
	StringLeft, url, post, InStr( post, "</image_link>" ) - 1
	StringTrimLeft, url, url, InStr( url, ">", 0, 0 )
	msgbox % Clipboard := url
Exitapp

makeProperBoundary(){
   Loop,26
      n .= chr(64+a_index)
   n .= "0123456789"
   Loop,% StrLen(A_Now) {
      Random,rnd,1,% StrLen(n)
      Random,UL,0,1
      b .= RegExReplace(SubStr(n,rnd,1),".$","$" (round(UL)? "U":"L") "0")
   }
   Return b
}

MimeType(ByRef Binary) {
   MimeTypes:="424d image/bmp|4749463 image/gif|ffd8ffe image/jpeg|89504e4 image/png|4657530"
          . " application/x-shockwave-flash|49492a0 image/tiff"
   @:="0123456789abcdef"
   Loop,8
      hex .= substr(@,(*(a:=&Binary-1+a_index)>>4)+1,1) substr(@,((*a)&15)+1,1)
   Loop,Parse,MimeTypes,|
      if ((substr(hex,1,strlen(n:=RegExReplace(A_Loopfield,"\s.*"))))=n)
         Mime := RegExReplace(A_LoopField,".*?\s")
   Return (Mime!="") ? Mime : "application/octet-stream"
}

For unicode, this is what is requred:
; Unicode
	FileGetSize, size, % file
	FileRead, img, % "*" file
	VarSetCapacity( data, size + 4096, 0 )
	itypea := NumGet( img, 0, "UShort" )
	itypeb := NumGet( img, 2, "UShort" )
	mimetype := itypea = 0x4d42 ? "image/bmp"
	: itypea = 0x4947 && itypeb = 0x3846 ? "image/gif"
	: itypea = 0xd8ff && itypeb = 0xe1ff ? "image/jpeg"
	: itypea = 0x5089 && itypeb = 0x474e ? "image/png"
	: itypea = 0x4949 && itypeb = 0x2a ? "image/tiff"
	: itypea = 0x5746 && itypeb = 0x53 ? "application/x-shockwave-flash"
	: "application/octet-stream"

	boundary := HTTPRequest_MD5( boundary := file "?" A_Now A_MSec, StrLen( file ) + 18 << !! A_IsUnicode )
	headers := "Content-type: multipart/form-data, boundary=" boundary "`nContent-Length: "
	file := RegexReplace( "\" file, ".*\\" )
	string =
	( LTRIM 
		--%boundary%
		content-disposition: form-data; name="MAX_FILE_SIZE"

		1048576
		--%boundary%
		content-disposition: form-data; name="xml"

		yes
		--%boundary%
		content-disposition: form-data; name="fileupload"; filename="%file%"
     	Content-type: %mimetype%
		Content-Transfer-Encoding: binary`n`n
	)

	offset := StrPut( string, &data, StrLen( string ), "UTF-8" )
	DllCall( "RtlMoveMemory", "Ptr", &data + offset, "Ptr", &img, "Int", size )
	string := "`n--" boundary "--"
	size += offset + StrPut( string, &data + offset + size, StrLen( string ), "UTF-8" )

	s := HTTPRequest( "http://www.imageshack.us/index.php", data, headers .= size )
	StringLeft, url, data, InStr( post, "</image_link>" ) - 1
	StringTrimLeft, url, url, InStr( url, ">", 0, 0 )
	msgbox % Clipboard := url
Exitapp
The returned headers and data should look like this:

HTTP/1.1 200 OK
Server: nginx/0.8.37
Date: Sun, 16 Oct 2011 22:57:30 GMT
Content-Type: text/xml
Transfer-Encoding: chunked
Connection: keep-alive
X-Powered-By: PHP/5.2.9
Set-Cookie: new_imageshack=y; expires=Tue, 24-Aug-2021 22:57:29 GMT; path=/; domain=.imageshack.us
Set-Cookie: ulevel=deleted; expires=Sat, 16-Oct-2010 22:57:28 GMT; path=/; domain=.imageshack.us
Set-Cookie: PHPSESSID=1d13659e298529db6d8f1e5cb7dda890; path=/
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Set-Cookie: latest=rdag4; expires=Wed, 10-Oct-2012 22:57:29 GMT; path=/; domain=.imageshack.us
Set-Cookie: latest=img580; expires=Wed, 10-Oct-2012 22:57:30 GMT; path=/; domain=.imageshack.us
Set-Cookie: new_imageshack=y; expires=Tue, 24-Aug-2021 22:57:30 GMT; path=/; domain=.imageshack.us
Set-Cookie: ulevel=deleted; expires=Sat, 16-Oct-2010 22:57:29 GMT; path=/; domain=.imageshack.us
Set-Cookie: PHPSESSID=c3aa57067e173a97d45128f40d8c3dfd; path=/
Set-Cookie: conversion=deleted; expires=Sat, 16-Oct-2010 22:57:29 GMT; path=/; domain=.imageshack.us
Set-Cookie: always_opt=-1; path=/; domain=.imageshack.us
Set-Cookie: rem_bar=0; path=/; domain=.imageshack.us
Pragma: public
Cache-Control: must-revalidate, post-check=0, pre-check=0

<?xml version="1.0" encoding="iso-8859-1"?><links>
<image_link>http://img580.images...ng</image_link>
<thumb_link>http://img580.images...ng</thumb_link>
<ad_link>http://img580.images...1.png</ad_link>
<thumb_exists>yes</thumb_exists>
<total_raters>0</total_raters>
<ave_rating>0.0</ave_rating>
<image_location>img580/2523/quirrel091.png</image_location>
<thumb_location>img580/2523/quirrel091.th.png</thumb_location>
<server>img580</server>
<image_name>quirrel091.png</image_name>
<done_page>http://img580.images...png</done_page>
<resolution>320x300</resolution>
<filesize>120018</filesize>
<image_class>r</image_class>
</links>

I suppose I could tweak HTTPRequest to allow binary file injection into multipart/form-data type POST. But before I do, I would like anyone who does use, or wants to use, HTTPRequest with multipart/form-data to reply so I can get an idea of how it is typically used.

nimda
  • Members
  • 4368 posts
  • Last active: Aug 09 2015 02:36 AM
  • Joined: 26 Dec 2010
Thank you very, very, very much! :D I would literally never have figured that out.

So should I wait until you tweak it, since the tweak would simplify the code involved and make it flavor-neutral?

VxE
  • Moderators
  • 3622 posts
  • Last active: Dec 24 2015 02:21 AM
  • Joined: 07 Oct 2006
No, don't wait; I really have no idea what the tweak might look like, whether it may allow multiple files, or permit split files. I'm not sure even how useful it might be.

The "unicode" version ought to work with ANSI AHK too, only it requires StrPut() (native to AHK-L). If you don't want to support AHK-Basic, go ahead and use that.

nimda
  • Members
  • 4368 posts
  • Last active: Aug 09 2015 02:36 AM
  • Joined: 26 Dec 2010
Edit: nice image btw

No, don't wait; I really have no idea what the tweak might look like, whether it may allow multiple files, or permit split files. I'm not sure even how useful it might be.

OK, I won't wait :)

The "unicode" version ought to work with ANSI AHK too

Amazing :shock:

only it requires StrPut() (native to AHK-L). If you don't want to support AHK-Basic, go ahead and use that.

I've already added a good deal of AHK_L code to Zizorz :p Your function is a golden example of how to maintain compatibility though, and that continues to impress me.

Once again, thank you!

nimda
  • Members
  • 4368 posts
  • Last active: Aug 09 2015 02:36 AM
  • Joined: 26 Dec 2010
Oops, I spoke too soon. Running your Unicode code gives me a blank MsgBox (yes I did verify the file exists.)

Changing the MsgBox to show the data, I get this:

---------------------------
test.ahk
---------------------------

<links>
<error id="wrong_file_type">Wrong file type detected for file green.jpg:application/octet-stream</error>
</links>

HTTP/1.1 200 OK
Server: nginx/0.8.37
Date: Tue, 18 Oct 2011 01:21:11 GMT
Content-Type: text/xml
Transfer-Encoding: chunked
Connection: keep-alive
X-Powered-By: PHP/5.2.9
Set-Cookie: new_imageshack=y; expires=Thu, 26-Aug-2021 01:21:11 GMT; path=/; domain=.imageshack.us
Set-Cookie: ulevel=deleted; expires=Mon, 18-Oct-2010 01:21:10 GMT; path=/; domain=.imageshack.us
Set-Cookie: PHPSESSID=cd502c91152ad5c176209c9a546380ae; path=/
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Cache-Control: no-store, no-cache, must-revalidate, post-check=0, pre-check=0
Pragma: no-cache
Set-Cookie: latest=img631; expires=Fri, 12-Oct-2012 01:21:11 GMT; path=/; domain=.imageshack.us
Set-Cookie: latest=img442; expires=Fri, 12-Oct-2012 01:21:11 GMT; path=/; domain=.imageshack.us


---------------------------
OK
---------------------------



[VxE] nli
  • Guests
  • Last active:
  • Joined: --
It's probably because of the following code:
mimetype := itypea = 0x4d42 ? "image/bmp"
   : itypea = 0x4947 && itypeb = 0x3846 ? "image/gif"
   : itypea = 0xd8ff && itypeb = 0xe1ff ? "image/jpeg"
   : itypea = 0x5089 && itypeb = 0x474e ? "image/png"
   : itypea = 0x4949 && itypeb = 0x2a ? "image/tiff"
   : itypea = 0x5746 && itypeb = 0x53 ? "application/x-shockwave-flash"
   : "application/octet-stream"
It's basically a hacked unwrap of the original 'MimeType' function. I didn't test it with every file type.

I'm pretty sure you just need to set 'mimetype' to the correct 'image/?' string accotding to the image type. Here's an untested alternative:
mimetype := RegexReplace( file, ".*\.(.*)", "image/$L1" )


m3nth0l
  • Members
  • 27 posts
  • Last active: Feb 13 2015 11:34 PM
  • Joined: 19 Oct 2011
Hey, I LOVE this function! Thanks so much for the hard work! I am running into a bit of a snag... I was hoping someone could help me out?

I'm attempting to log into a device, however the page has a certificate error. The headers return A_LastError = 12045. Per the MSDN site, it states ERROR_INTERNET_INVALID_CA

Normally I can get around this cert error with COM, but I'm trying to collect all the html via Get, as It is very fast. I suppose that I could just use COM in a hidden window, but I'd like to avoid that if possible.

Is it possible to implement something like this http://support.microsoft.com/kb/182888with httpRequest to bypass this error?

Thanks guys!

nimda
  • Members
  • 4368 posts
  • Last active: Aug 09 2015 02:36 AM
  • Joined: 26 Dec 2010

I'm pretty sure you just need to set 'mimetype' to the correct 'image/?' string accotding to the image type. Here's an untested alternative:

mimetype := RegexReplace( file, ".*\.(.*)", "image/$L1" )

Well I run this code:
FileSelectFile File
; Unicode
   FileGetSize, size, % file
   FileRead, img, % "*" file
   VarSetCapacity( data, size + 4096, 0 )
   mimetype := RegexReplace( file, ".*\.(.*)", "image/$L1" )

   boundary := HTTPRequest_MD5( boundary := file "?" A_Now A_MSec, StrLen( file ) + 18 << !! A_IsUnicode )
   headers := "Content-type: multipart/form-data, boundary=" boundary "`nContent-Length: "
   file := RegexReplace( "" file, ".*\" )
   string =
   ( LTRIM 
      --%boundary%
      content-disposition: form-data; name="MAX_FILE_SIZE"

      1048576
      --%boundary%
      content-disposition: form-data; name="xml"

      yes
      --%boundary%
      content-disposition: form-data; name="fileupload"; filename="%file%"
        Content-type: %mimetype%
      Content-Transfer-Encoding: binary`n`n
   )

   offset := StrPut( string, &data, StrLen( string ), "UTF-8" )
   DllCall( "RtlMoveMemory", "Ptr", &data + offset, "Ptr", &img, "Int", size )
   string := "`n--" boundary "--"
   size += offset + StrPut( string, &data + offset + size, StrLen( string ), "UTF-8" )

   s := HTTPRequest( "http://www.imageshack.us/index.php", data, headers .= size )
   StringLeft, url, data, InStr( post, "</image_link>" ) - 1
   StringTrimLeft, url, url, InStr( url, ">", 0, 0 )
   msgbox % Clipboard := url
Exitapp
but no matter what image file I select, it gives me a blank MsgBox. :cry:

VxE
  • Moderators
  • 3622 posts
  • Last active: Dec 24 2015 02:21 AM
  • Joined: 07 Oct 2006

Is it possible to implement something like this http://support.microsoft.com/kb/182888with httpRequest to bypass this error?

If you don't mind being the guinea pig, copy the following lines into the HTTPRequest code just after the block under the call to 'HttpAddRequestHeaders' (around line #507)
If DllCall( "InternetQueryOption" WorA, pty, hRequest, "UInt", 31, "UInt*", Flags, "UInt", 4 )
		DllCall( "InternetSetOption" WorA, pty, hRequest, "UInt", 31, "UInt*", Flags |= 0x100, "UInt", 4 )
and let me know if it works.



@ nimda, does it give you the 'wrong_file_type' message every time? Have you tried with different file types? What about manually setting the 'mimetype' to 'image/jpeg'?

m3nth0l
  • Members
  • 27 posts
  • Last active: Feb 13 2015 11:34 PM
  • Joined: 19 Oct 2011

If you don't mind being the guinea pig, copy the following lines into the HTTPRequest code just after the block under the call to 'HttpAddRequestHeaders' (around line #507)

If DllCall( "InternetQueryOption" WorA, pty, hRequest, "UInt", 31, "UInt*", Flags, "UInt", 4 )
		DllCall( "InternetSetOption" WorA, pty, hRequest, "UInt", 31, "UInt*", Flags |= 0x100, "UInt", 4 )
and let me know if it works.



Yeah, I don't mind being a guinea pig at all - I appreciate it! I added the code right at line 507 right after the httpRequestHeaders, but I'm still getting the 12045 error. If you've got any other suggestions I'd be glad to test them. Thanks again for the help, and the awesome function! The xmlMakePretty alone has already saved my eyesight and improved my well being beyond belief!

VxE
  • Moderators
  • 3622 posts
  • Last active: Dec 24 2015 02:21 AM
  • Joined: 07 Oct 2006
My bad, I didn't even check to see if those functions returned without errors. This is how it should be:
If DllCall( "WinINet\InternetQueryOption" WorA, pty, hRequest, "UInt", 31, "UInt*", pos, "UInt*", 4 )

		DllCall( "WinINet\InternetSetOption" WorA, pty, hRequest, "UInt", 31, "UInt*", pos |= 0x100, "UInt*", 4 )


nimda
  • Members
  • 4368 posts
  • Last active: Aug 09 2015 02:36 AM
  • Joined: 26 Dec 2010
[VxE], this code works:
FileSelectFile File
; Unicode
   FileGetSize, size, % file
   FileRead, img, % "*" file
   VarSetCapacity( data, size + 4096, 0 )
   mimetype := "image/jpeg"

   boundary := HTTPRequest_MD5( boundary := file "?" A_Now A_MSec, StrLen( file ) + 18 << !! A_IsUnicode )
   headers := "Content-type: multipart/form-data, boundary=" boundary "`nContent-Length: "
   file := RegexReplace( "\" file, ".*\\" )
   string =
   ( LTRIM 
      --%boundary%
      content-disposition: form-data; name="MAX_FILE_SIZE"

      1048576
      --%boundary%
      content-disposition: form-data; name="xml"

      yes
      --%boundary%
      content-disposition: form-data; name="fileupload"; filename="%file%"
        Content-type: %mimetype%
      Content-Transfer-Encoding: binary`n`n
   )

   offset := StrPut( string, &data, StrLen( string ), "UTF-8" )
   DllCall( "RtlMoveMemory", "Ptr", &data + offset, "Ptr", &img, "Int", size )
   string := "`n--" boundary "--"
   size += offset + StrPut( string, &data + offset + size, StrLen( string ), "UTF-8" )

   s := HTTPRequest( "http://www.imageshack.us/index.php", data, headers .= size )
   StringLeft, url, data, InStr( post, "</image_link>" ) - 1
   StringTrimLeft, url, url, InStr( url, ">", 0, 0 )
   msgbox % data
Exitapp
:)
Thank you!