使用spark sql读取动态分区修剪与hive表

时间:2017-08-10 14:47:18

标签: apache-spark-sql

我正在尝试使用<?xml version='1.0' encoding='utf-8'?> <manifest android:hardwareAccelerated="true" android:versionCode="1" android:versionName="0.0.1" package="mypackage" xmlns:android="http://schemas.android.com/apk/res/android"> <supports-screens android:anyDensity="true" android:largeScreens="true" android:normalScreens="true" android:resizeable="true" android:smallScreens="true" android:xlargeScreens="true" /> <uses-permission android:name="android.permission.INTERNET" /> <application android:hardwareAccelerated="true" android:icon="@mipmap/icon" android:label="@string/app_name" android:supportsRtl="true"> <activity android:configChanges="orientation|keyboardHidden|keyboard |screenSize|locale" android:label="@string/activity_name" android:launchMode="singleTask" android:name="MainActivity" android:theme="@android:style/Theme.DeviceDefault.NoActionBar" android:windowSoftInputMode="adjustResize"> <intent-filter> <action android:name="android.intent.action.MAIN" /> <category android:name="android.intent.category.LAUNCHER" /> </intent-filter> <intent-filter> <action android:name="android.intent.action.VIEW" /> <category android:name="android.intent.category.DEFAULT" /> <category android:name="android.intent.category.BROWSABLE" /> <data android:host="myhost" android:scheme="myscheme"/> </intent-filter> </activity> <meta-data android:name="com.facebook.sdk.ApplicationId" android:value="@string/fb_app_id" /> <meta-data android:name="com.facebook.sdk.ApplicationName" android:value="@string/fb_app_name" /> <activity android:configChanges="keyboard|keyboardHidden|screenLayout |screenSize|orientation" android:label="@string/fb_app_name" android:name="com.facebook.FacebookActivity" android:theme="@android:style/Theme.Translucent.NoTitleBar" /> <receiver android:exported="false" android:name="de.appplant.cordova.plugin .localnotification.TriggerReceiver" /> <receiver android:exported="false" android:name= "de.appplant.cordova.plugin.localnotification.ClearReceiver" /> <activity android:exported="false" android:launchMode="singleTask" android:name="de.appplant.cordova.plugin.localnotification.ClickActivity" android:theme="@android:style/Theme.NoDisplay" /> <receiver android:exported="false" android:name="de.appplant.cordova.plugin.notification.TriggerReceiver" /> <receiver android:exported="false" android:name="de.appplant.cordova.plugin.notification.ClearReceiver" /> <receiver android:exported="false" android:name="de.appplant.cordova.plugin.localnotification.RestoreReceiver"> <intent-filter> <action android:name="android.intent.action.BOOT_COMPLETED" /> </intent-filter> </receiver> <activity android:exported="false" android:launchMode="singleTask" android:name="de.appplant.cordova.plugin.notification.ClickActivity" android:theme="@android:style/Theme.NoDisplay" /> </application> <uses-sdk android:minSdkVersion="16" android:targetSdkVersion="25" /> <uses-permission android:name="android.permission.ACCESS_COARSE_LOCATION" /> <uses-permission android:name="android.permission.ACCESS_FINE_LOCATION" /> <uses-permission android:name="android.permission.ACCESS_LOCATION_EXTRA_COMMANDS" /> <uses-permission android:name="android.permission.CAMERA" /> <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" /> <uses-permission android:name="android.permission.RECORD_AUDIO" /> <uses-permission android:name="android.permission.WAKE_LOCK" /> <uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" /> <uses-feature android:name="android.hardware.camera" android:required="true" /> <uses-feature android:name="android.hardware.camera.autofocus" android:required="true" /> <uses-permission android:name="android.permission.READ_CONTACTS" /> <uses-permission android:name="android.permission.WRITE_CONTACTS" /> <uses-permission android:name="android.permission.GET_ACCOUNTS" /> <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" /> <uses-permission android:name="android.permission.ACCESS_WIFI_STATE" /> <uses-permission android:name="android.permission.READ_PHONE_STATE" /> <uses-permission android:name="android.permission.RECEIVE_BOOT_COMPLETED" /> Spark SQL阅读Hive分区表 经过长时间的搜索后,我注意到Spark 1.6.中没有动态分区修剪功能。

对于以下sql,它将扫描所有分区,

Spark 1.6

on(Select col1, col2, col3, col4 from table join dimension ) 其中dimension.partitn_col = 1

问:在Spark 1.6中是否有针对此问题的解决方案/解决方法?

0 个答案:

没有答案