From 0b02e01474d077f2bc05cc93564b4df16d3b67e9 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 27 Feb 2018 18:17:44 -0500 Subject: [PATCH] whoot Signed-off-by: Kenneth Reitz --- Pipfile | 15 +++ Pipfile.lock | 304 ++++++++++++++++++++++++++++++++++++++++++++++++ pysoundcloud.py | 131 +++++++++++++++++++++ 3 files changed, 450 insertions(+) create mode 100644 Pipfile create mode 100644 Pipfile.lock create mode 100644 pysoundcloud.py diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..19ebc63 --- /dev/null +++ b/Pipfile @@ -0,0 +1,15 @@ +[[source]] + +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + + +[packages] + +requests-html = "*" +maya = "*" + + +[dev-packages] + diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..c735be7 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,304 @@ +{ + "_meta": { + "hash": { + "sha256": "c0f18c6e898b62c5863dea124e7898b07241179e1cfc301eb09079ba84775a38" + }, + "host-environment-markers": { + "implementation_name": "cpython", + "implementation_version": "3.6.4", + "os_name": "posix", + "platform_machine": "x86_64", + "platform_python_implementation": "CPython", + "platform_release": "17.4.0", + "platform_system": "Darwin", + "platform_version": "Darwin Kernel Version 17.4.0: Sun Dec 17 09:19:54 PST 2017; root:xnu-4570.41.2~1/RELEASE_X86_64", + "python_full_version": "3.6.4", + "python_version": "3.6", + "sys_platform": "darwin" + }, + "pipfile-spec": 6, + "requires": {}, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.python.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "beautifulsoup4": { + "hashes": [ + "sha256:7015e76bf32f1f574636c4288399a6de66ce08fb7b2457f628a8d70c0fbabb11", + "sha256:11a9a27b7d3bddc6d86f59fb76afb70e921a25ac2d6cc55b40d072bd68435a76", + "sha256:808b6ac932dccb0a4126558f7dfdcf41710dd44a4ef497a0bb59a77f9f078e89" + ], + "version": "==4.6.0" + }, + "bs4": { + "hashes": [ + "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a" + ], + "version": "==0.0.1" + }, + "certifi": { + "hashes": [ + "sha256:14131608ad2fd56836d33a71ee60fa1c82bc9d2c8d98b7bdbc631fe1b3cd1296", + "sha256:edbc3f203427eef571f79a7692bb160a2b0f7ccaa31953e99bd17e307cf63f7d" + ], + "version": "==2018.1.18" + }, + "chardet": { + "hashes": [ + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691", + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae" + ], + "version": "==3.0.4" + }, + "cssselect": { + "hashes": [ + "sha256:3b5103e8789da9e936a68d993b70df732d06b8bb9a337a05ed4eb52c17ef7206", + "sha256:066d8bc5229af09617e24b3ca4d52f1f9092d9e061931f4184cd572885c23204" + ], + "version": "==1.0.3" + }, + "dateparser": { + "hashes": [ + "sha256:b452ef8b36cd78ae86a50721794bc674aa3994e19b570f7ba92810f4e0a2ae03", + "sha256:940828183c937bcec530753211b70f673c0a9aab831e43273489b310538dff86" + ], + "version": "==0.7.0" + }, + "fake-useragent": { + "hashes": [ + "sha256:cc9b9ddcebc708b3deac846f5fccb16e37c02ee47435a4ec7132271dd96aec8c" + ], + "version": "==0.1.10" + }, + "humanize": { + "hashes": [ + "sha256:a43f57115831ac7c70de098e6ac46ac13be00d69abbf60bdcac251344785bb19" + ], + "version": "==0.5.1" + }, + "idna": { + "hashes": [ + "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4", + "sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f" + ], + "version": "==2.6" + }, + "lxml": { + "hashes": [ + "sha256:41f59cbdab232f11680d5d4dec9f2e6782fd24d78e37ee833447702e34e675f4", + "sha256:e7e41d383f19bab9d57f5f3b18d158655bcd682e7e723f441b9e183e1e35a6b5", + "sha256:155521c337acecf8202091cff85bb9f709f238130ebadf04280fb1db11f5ad8b", + "sha256:d2c985d2460b81c6ca5feb8b86f1bc594ad59405d0bdf68626b85852b701553c", + "sha256:950e63387514aa1b881eba5ac6cb2ec51a118b3dafe99dd80ca19d8fb0142f30", + "sha256:470d7ce41e8047208ba1a376560bad17f1468df1f3097bc83902b26cfafdbb0c", + "sha256:e608839a5ee2180164424ccf279c8e2d9bbe8816d002c58fd97d6b621ba4aa94", + "sha256:87a66bcadac270fc010cb029022a93fc722bf1204a8b03e782d4c790f0edf7ca", + "sha256:2dedfeeecc2d5a939cf622602f5a1ce443ca82407f386880f739f1a9f08053ad", + "sha256:ba05732e4bcf59e948f61588851dcf620fd60d5bbd9d704203e5f59bbaa60219", + "sha256:2190266059fec3c5a55f9d6c30532c64c6d414d3228909c0af573fe4907e78d1", + "sha256:dd291debfaa535d9cb6cee8d7aca2328775e037d02d13f1634e57f49bc302cc4", + "sha256:29a36e354c39b2e24bc4ee103de53417ebb80f976a6ab9e8d093d559e2ac03e1", + "sha256:e37427d5a27eefbcfc48847e0b37f348113fac7280bc857421db39ffc6372570", + "sha256:b106d4d2383382399ad82108fd187e92f40b1c90f55c2d36bbcb1c44bcf940fc", + "sha256:0ee07da52d240f1dc3c83eef5cd5f1b7f018226c1121f2a54d446645779a6d17", + "sha256:3b33549fb8f91b38a7500078242b03cca513f3412a2cdae722e89bf83f95971d", + "sha256:4c12e90886d9c53ab434c8d0cebea122321cce19614c3c6b6d1a7700d7cc6212", + "sha256:79322000279cda10b53c374d53ca632ead3bc51c6aebf8e62c8fa93a4d08b750", + "sha256:6cba398eb37e0631e60e0e080c101cfe91769b2c8267105b64b4625e2581ea21", + "sha256:49a655956f8de69e1258bc0fcfc43eb3bd1e038655784d77d1869b4b81444e37", + "sha256:af8a5373241d09b8fc53e0490e1719ce5dc90a21b19db89b6596c1adcdd52270", + "sha256:e6b6698415c7e8d227a47a3b1038e1b37c2b438a1b48c2db7ad9e74ddbcd1149", + "sha256:155c916cf2645b4a8f2bd5d09065e92d1b67b8d464bdc001e0b524af84bedf6f", + "sha256:fa7320679ced5e25b20203d157280680fc84eb783b6cc650cb0c98e1858b7dd3", + "sha256:4187c4b0cefc3353181db048c51f42c489d9ac51e40b86c4851dc0671372971d", + "sha256:d5d29663e979e83b3fc361e97200f959cddb3a14797391d15273d84a5a8ae44b", + "sha256:940caef1ec7c78e0c34b0f6b94fe42d0f2022915ffc78643d28538a5cfd0f40e" + ], + "version": "==4.1.1" + }, + "maya": { + "hashes": [ + "sha256:ad1969bae78afb148c45a2f63591a7575ec05b4a0ab7ec04987ab7d73649f9d6", + "sha256:d8a7ed8513b2990036fe456c9f595b54d19ec49cb4461cd95a2ef6c487fb55eb" + ], + "version": "==0.3.4" + }, + "parse": { + "hashes": [ + "sha256:8048dde3f5ca07ad7ac7350460952d83b63eaacecdac1b37f45fd74870d849d2" + ], + "version": "==1.8.2" + }, + "pendulum": { + "hashes": [ + "sha256:881efe37328de0785c0731d462e1485a45712f2cd5cb55907d6c15458460ebeb", + "sha256:3c85e8cbc91f45e1cc916cc9180b34153cd6aaaaacfb51a48b3156318314fa82", + "sha256:0c14388546db6605a860b8b7112cb69d0b11c9ce5e072210504544e0d4575799", + "sha256:8798aeca58b3dd7ffdc5a4993c9eaafedc4048165429e8f499ddd62c73bf3964", + "sha256:8199206c479b13947dcac63c025575d035331bb3819d1783dc1d568a11962906", + "sha256:bcca072f82e84b419efec1320cd3ee5c230d263f3a601b146651ed4db77d89f0", + "sha256:ff0c5fa3af4a471a218408c448b804ac6bccb105127727474f4e83c0e4072e97", + "sha256:39a255776528afe11ea0d57814f9bf3729c1e0b99063af2e5c6cfd750c3e1f7f" + ], + "version": "==1.4.2" + }, + "pyee": { + "hashes": [ + "sha256:47f8fa96d6dee61c82001831e1fbba55f3f808003a322d0e6653aa01c59f6b9e", + "sha256:4ec22817297b7024f89721cc34f790ee2767c5b5ca44284c565ee643abafbe32" + ], + "version": "==5.0.0" + }, + "pyppeteer": { + "hashes": [ + "sha256:596929fb7d052048679081d3dc2a998cf065e936a752c7ba2392445d6e0e9706" + ], + "version": "==0.0.10" + }, + "pyquery": { + "hashes": [ + "sha256:07987c2ed2aed5cba29ff18af95e56e9eb04a2249f42ce47bddfb37f487229a3", + "sha256:4771db76bd14352eba006463656aef990a0147a0eeaf094725097acfa90442bf" + ], + "version": "==1.4.0" + }, + "python-dateutil": { + "hashes": [ + "sha256:95511bae634d69bc7329ba55e646499a842bc4ec342ad54a8cdb65645a0aad3c", + "sha256:891c38b2a02f5bb1be3e4793866c8df49c7d19baabf9c1bad62547e0b4866aca" + ], + "version": "==2.6.1" + }, + "pytz": { + "hashes": [ + "sha256:ed6509d9af298b7995d69a440e2822288f2eca1681b8cce37673dbb10091e5fe", + "sha256:f93ddcdd6342f94cea379c73cddb5724e0d6d0a1c91c9bdef364dc0368ba4fda", + "sha256:61242a9abc626379574a166dc0e96a66cd7c3b27fc10868003fa210be4bff1c9", + "sha256:ba18e6a243b3625513d85239b3e49055a2f0318466e0b8a92b8fb8ca7ccdf55f", + "sha256:07edfc3d4d2705a20a6e99d97f0c4b61c800b8232dc1c04d87e8554f130148dd", + "sha256:3a47ff71597f821cd84a162e71593004286e5be07a340fd462f0d33a760782b5", + "sha256:5bd55c744e6feaa4d599a6cbd8228b4f8f9ba96de2c38d56f08e534b3c9edf0d", + "sha256:887ab5e5b32e4d0c86efddd3d055c1f363cbaa583beb8da5e22d2fa2f64d51ef", + "sha256:410bcd1d6409026fbaa65d9ed33bf6dd8b1e94a499e32168acfc7b332e4095c0" + ], + "version": "==2018.3" + }, + "pytzdata": { + "hashes": [ + "sha256:7cd949123e2c2060fd12793de3a4a449e36b5dea5e169b810a3ac3f0b9877cfa", + "sha256:4e2cceb54335cd6c28caea46b15cd592e2aec5e8b05b0241cbccfb1b23c02ae7" + ], + "version": "==2018.3" + }, + "regex": { + "hashes": [ + "sha256:333687d9a44738c486735955993f83bd22061a416c48f5a5f9e765e90cf1b0c9", + "sha256:361a1fd703a35580a4714ec28d85e29780081a4c399a99bbfb2aee695d72aedb", + "sha256:f69d1201a4750f763971ea8364ed95ee888fc128968b39d38883a72a4d005895", + "sha256:a50532f61b23d4ab9d216a6214f359dd05c911c1a1ad20986b6738a782926c1a", + "sha256:1b428a296531ea1642a7da48562746309c5c06471a97bd0c02dd6a82e9cecee8", + "sha256:5b9c0ddd5b4afa08c9074170a2ea9b34ea296e32aeea522faaaaeeeb2fe0af2e", + "sha256:27d72bb42dffb32516c28d218bb054ce128afd3e18464f30837166346758af67", + "sha256:32cf4743debee9ea12d3626ee21eae83052763740e04086304e7a74778bf58c9", + "sha256:35eeccf17af3b017a54d754e160af597036435c58eceae60f1dd1364ae1250c7", + "sha256:be42a601aaaeb7a317f818490a39d153952a97c40c6e9beeb2a1103616405348", + "sha256:eee4d94b1a626490fc8170ffd788883f8c641b576e11ba9b4a29c9f6623371e0", + "sha256:32f6408dbca35040bc65f9f4ae1444d5546411fde989cb71443a182dd643305e", + "sha256:a9243d7b359b72c681a2c32eaa7ace8d346b7e8ce09d172a683acf6853161d9c", + "sha256:494bed6396a20d3aa6376bdf2d3fbb1005b8f4339558d8ac7b53256755f80303", + "sha256:b44624a38d07d3c954c84ad302c29f7930f4bf01443beef5589e9157b14e2a29" + ], + "version": "==2018.2.21" + }, + "requests": { + "hashes": [ + "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b", + "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e" + ], + "version": "==2.18.4" + }, + "requests-html": { + "hashes": [ + "sha256:731bc49f3d042b9b5989d62e285019532be8a4ea44218d350874c4bb998fcf6b", + "sha256:a694c73cbc13b16a0f00e0db4cce435a1fd60c1dae39fed58b1c3f2ccb9522a3" + ], + "version": "==0.6.3" + }, + "ruamel.yaml": { + "hashes": [ + "sha256:14d161558e3bf89e87d77c218098be22fa9a0d6d0bea40250fce525b1d0cbee2", + "sha256:fcfc24a21594c071cc4588e84b7657a1f47ebcf6037c6c43fa15c4bbd3989ec2", + "sha256:02babffd019911841ba01b76e23dfec7c9e9b2725503fb2698c4982fa1a6e835", + "sha256:c0908896e34b617ead40552cab03c1769bdc43d1da02419160dc900c5dfddde2", + "sha256:01e30ecb1b1c0ebf9fce814dc20dace402571517277799291202b61b22096c24", + "sha256:b6c5d5f03ba78e3f27c7188a00c4e09b6a4507fe3154ba40a294e09cb30ee016", + "sha256:9225c83952d28f302cfc23c3d9a6f8231bfd581476d7aff1e3c7de49eecb4ee9", + "sha256:c41e04b526d0153c9246cfab87d7ddefdc9f165cb8886a8ec48ba7a2b73069f6", + "sha256:6d05c5a5baf829c70916c226ef3200650846a7227de226bca8a59efaf88bb973", + "sha256:e3bbfe0d294e08fdbb0cb05485435a2ceb4e168e98b5dc611f051c1864986b4b", + "sha256:68c8f2986bcb91b6db1aea8698941769840c7257e951a9377048f7eff35be773", + "sha256:072f6364a89972e8dc0afdce3335a709d5464dfeaa4f736d092a54574338b874", + "sha256:5504398fc755a2b14c9983b2101161a8591a4b30812590cc1c365e7fcc117dfa", + "sha256:e2d2715bf92156bec5fb42e92e95dac1c4d9904f8a3d4e2d0c438758fe9092d7", + "sha256:6d7929b24e329d662fa43b657fddfee5260e2d35d0a543065cd755d4e17a9b2f", + "sha256:f2d02a4af5a13b09d0b823cdd0317b54f3e0115e50b5ac4d9840c3a1b566817f", + "sha256:8dc74821e4bb6b21fb1ab35964e159391d99ee44981d07d57bf96e2395f3ef75" + ], + "version": "==0.15.35" + }, + "six": { + "hashes": [ + "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb", + "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9" + ], + "version": "==1.11.0" + }, + "tzlocal": { + "hashes": [ + "sha256:4ebeb848845ac898da6519b9b31879cf13b6626f7184c496037b818e238f2c4e" + ], + "version": "==1.5.1" + }, + "urllib3": { + "hashes": [ + "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b", + "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f" + ], + "version": "==1.22" + }, + "w3lib": { + "hashes": [ + "sha256:aaf7362464532b1036ab0092e2eee78e8fd7b56787baa9ed4967457b083d011b", + "sha256:55994787e93b411c2d659068b51b9998d9d0c05e0df188e6daf8f45836e1ea38" + ], + "version": "==1.19.0" + }, + "websockets": { + "hashes": [ + "sha256:f5192da704535a7cbf76d6e99c1ec4af7e8d1288252bf5a2385d414509ded0cf", + "sha256:0c31bc832d529dc7583d324eb6c836a4f362032a1902723c112cf57883488d8c", + "sha256:da7610a017f5343fdf765f4e0eb6fd0dfd08264ca1565212b110836d9367fc9c", + "sha256:fd81af8cf3e69f9a97f3a6c0623a0527de0f922c2df725f00cd7646d478af632", + "sha256:3d425ae081fb4ba1eef9ecf30472ffd79f8e868297ccc7a47993c96dbf2a819c", + "sha256:ebdd4f18fe7e3bea9bd3bf446b0f4117739478caa2c76e4f0fb72cc45b03cbd7", + "sha256:3859ca16c229ddb0fa21c5090e4efcb037c08ce69b0c1dfed6122c3f98cd0c22", + "sha256:d1a0572b6edb22c9208e3e5381064e09d287d2a915f90233fef994ee7a14a935", + "sha256:80188abdadd23edaaea05ce761dc9a2e1df31a74a0533967f0dcd9560c85add0", + "sha256:fecf51c13195c416c22422353b306dddb9c752e4b80b21e0fa1fccbe38246677", + "sha256:367ff945bc0950ad9634591e2afe50bf2222bc4fad1088a386c4bb700888026e", + "sha256:6df87698022aef2596bffdfecc96d656db59c8d719708c8a471daa815ee61656", + "sha256:341824d8c9ad53fc43cca3fa9407f294125fa258592f7676640396501448e57e", + "sha256:64896a6b3368c959b8096b655e46f03dfa65b96745249f374bd6a35705cc3489", + "sha256:1f3e5a52cab6daa3d432c7b0de0a14109be39d2bfaad033ee5de4a3d3e11dcdf", + "sha256:da4d4fbe059b0453e726d6d993760065d69b823a27efc3040402a6fcfe6a1ed9" + ], + "version": "==4.0.1" + } + }, + "develop": {} +} diff --git a/pysoundcloud.py b/pysoundcloud.py new file mode 100644 index 0000000..5b4263d --- /dev/null +++ b/pysoundcloud.py @@ -0,0 +1,131 @@ +import maya +from requests_html import HTMLSession + +session = HTMLSession() + +class Track: + def __init__(self): + self.title = None + self.likes = None + self.posted = None + self.url = None + + def __repr__(self): + return "".format(self.url) + + @classmethod + def from_element(cls, *, sc, element): + track = cls() + + track.title = element.find('.soundTitle__title', first=True).text + try: + track.likes = int(element.find('button.sc-button-like.sc-button.sc-button-small.sc-button-responsive', first=True).html.split('title="Like">')[1].split('<')[0]) + except ValueError: + pass + track.posted = maya.parse(element.find('.relativeTime', first=True).attrs['datetime']).datetime + track.url = '{}{}'.format(sc.base_url, element.find('.soundTitle__title', first=True).attrs['href']) + + return track + + +class User: + def __init__(self, *, username, sc): + self.username = username + self.sc = sc + + self._meta = {} + + def __repr__(self): + return "".format(self.username) + + @property + def meta(self): + if self._meta: + return self._meta + + r = session.get(self.sc._construct_url(self.username)) + r.html.render() + + # Get metadata. + self._meta['username'] = r.html.find('.profileHeaderInfo__userName', first=True).text + try: + self._meta['additional'] = r.html.find('.profileHeaderInfo__additional', first=True).text + except AttributeError: + self._meta['additional'] = None + + try: + self._meta['bio'] = r.html.find('#content > div > div.l-fluid-fixed > div.l-sidebar-right.l-user-sidebar-right > div > article.infoStats > div.infoStats__description > div > div > div', first=True).text + except AttributeError: + self._meta['bio'] = None + + self._meta['followers'] = int(r.html.find('#content > div > div.l-fluid-fixed > div.l-sidebar-right.l-user-sidebar-right > div > article.infoStats > table > tbody > tr > td:nth-child(1) > a > div', first=True).text.replace(',', '').replace('K', '000')) + self._meta['following'] = int(r.html.find('#content > div > div.l-fluid-fixed > div.l-sidebar-right.l-user-sidebar-right > div > article.infoStats > table > tbody > tr > td:nth-child(2) > a > div', first=True).text.replace(',', '').replace('K', '000')) + self._meta['tracks'] = int(r.html.find('#content > div > div.l-fluid-fixed > div.l-sidebar-right.l-user-sidebar-right > div > article.infoStats > table > tbody > tr > td:nth-child(3) > a > div', first=True).text.replace(',', '').replace('K', '000')) + try: + self._meta['likes'] = int(r.html.find('#content > div > div.l-fluid-fixed > div.l-sidebar-right.l-user-sidebar-right > div > article.sidebarModule.g-all-transitions-200-linear.likesModule > a > h3 > span.sidebarHeader__actualTitle', first=True).text.split()[0].replace(',', '')) + except AttributeError: + self._meta['likes'] = None + + + return self._meta + + + @property + def following(self): + + def gen(): + r = session.get(self.sc._construct_url(self.username, 'following')) + + r.html.render(scrolldown=int(self.meta['following'] / 10), sleep=2) + + for user in r.html.find('a.userBadgeListItem__heading'): + username = user.attrs['href'][1:].strip() + yield User(username=username, sc=self.sc) + + return [g for g in gen()] + + @property + def followers(self): + def gen(): + r = session.get(self.sc._construct_url(self.username, 'followers')) + + r.html.render(scrolldown=int(self.meta['followers'] / 10), sleep=2) + + for user in r.html.find('a.userBadgeListItem__heading'): + username = user.attrs['href'][1:].strip() + yield User(username=username, sc=self.sc) + + return [g for g in gen()] + + @property + def tracks(self): + def gen(): + r = session.get(self.sc._construct_url(self.username, 'tracks')) + + r.html.render(scrolldown=int(self.meta['tracks']), sleep=1) + + for track in r.html.find('.sound__content'): + yield Track.from_element(sc=self.sc, element=track) + + return [g for g in gen()] + + +class SoundCloud: + def __init__(self): + self.base_url = 'https://soundcloud.com' + + def _construct_url(self, *frags): + return '{}/{}'.format(self.base_url, '/'.join(frags)) + + def get_user(self, username): + return User(username=username, sc=self) + + +sc = SoundCloud() +user = sc.get_user('kanyewest') +print(user.meta) +# print(alex.meta) +# print(alex.meta) +# for track in user.tracks: + # print(track.title) +# print(len(alex.following)) \ No newline at end of file