Question

写了一个火花代码，Google BQ使用https://github.com/GoogleCloudPlatform/spark-bigquery-connector进行了库规范中所述的更改。当我从IDE（Intellij）运行该作业时，它运行良好，并已连接到Bq。而当我尝试通过spark-submit运行时，它给出了以下错误。有什么线索我要去哪里吗？

<!DOCTYPE html>
<html>
<head>
<meta name="viewport" content="width=device-width" />
<title>Items</title>
<link href="~/Content/jquery.dataTables.min.css" rel="stylesheet" />
<link href="~/Content/themes/base/jquery-ui.min.css" rel="stylesheet" />
</head>
<body>
<div style="width:90%; margin:0 auto" class="tablecontainer">
    <a class="popup btn btn-primary" href="/home/CreateItem/0" style="margin-bottom:20px; margin-top:60px">Add new Item </a>
    <table id="CBR-Item">
        <thead>
            <tr style="color:lightgray">
                <th>Serial Number</th>
                <th>Safe ID</th>
                <th>Date of Entry</th>
                <th>Title/Subject</th>
                <th>Document type</th>
                <th>Sender of Originator</th>
                <th>Reference Number</th>
                <th>Protective Marking</th>
                <th>Number recieved/produced</th>
                <th>copy number</th>
                <th>Status</th>
                <th>Same-Day Loan</th>
                <th>Edit</th>
                <th>History</th>
            </tr>
        </thead>
    </table>
</div>

<script src="https://code.jquery.com/jquery-3.1.1.min.js"></script>
<script src="~/Scripts/date.format.js"></script>
<script src="~/Scripts/jquery.validate.min.js"></script>
<script src="~/Scripts/jquery.validate.unobtrusive.min.js"></script>
<script src="https://cdn.datatables.net/1.10.13/js/jquery.dataTables.min.js"></script>
<script src="~/Scripts/jquery-ui-1.12.1.min.js"></script>

<script>
    var isItCBO;
    $(document).ready(function () {  
        $.ajax({
            url: '@Url.RouteUrl(new{ action="CheckCBOStatus", controller="Home"})',
            type: 'GET',
            dataType: 'json',
            success: function (data) {
                isItCBO = data.data;
            },
            error: function (xhr) {
                alert("CBO Verification ERROR");
            }
        })
        var oTable = $('#CBR-Item').DataTable({
            "ajax": {
                "url": '/Home/GetSafeItems',
                "type": "get",
                "datatype": "json",
            },
            "columns": [

                { "data": "Serial_Number", "autoWidth": true },
                { "data": "Safe_ID", "autoWidth": true },
                {
                    "data": "Date_of_Entry", "autoWidth": true, "render": function (data) {
                        var date = new Date(parseInt(data.substr(6)));
                        var result = date.format("dd/mm/yyyy HH:MM:ss");
                        return result;
                    }
                },
                { "data": "Title_subject", "autoWidth": true },
                { "data": "Document_Type", "autoWidth": true },
                { "data": "Sender_of_Originator", "autoWidth": true },
                { "data": "Reference_Number", "autoWidth": true },
                { "data": "Protective_Marking", "autoWidth": true },
                { "data": "Number_recieved_produced", "autoWidth": true },
                { "data": "copy_number", "autoWidth": true },
                { "data": "Status", "autoWidth": true },
                { "data": "Same_day_Loan", "autoWidth": true },
                {
                    "data": "Serial_Number", "width": "50px", "render": function (data, type, row) {
                        if (isItCBO == true) {
                            return '<a class="popup" href="/home/SaveItem/' + data + '">Edit</a>';
                        }
                        else {
                            return '<a class="errorMe" href="" >Edit</a>';
                        }
                              } },

                {
                    "data": "Serial_Number", "width": "50px", "render": function (data, type, row) {
                        if (isItCBO == true) {
                            return '<a class="popup" href="/home/EventHistory/' + data + '" itemsel="' + data + '">History</a>';
                        }
                        else {
                            return '<a class="errorMe" href="" >History</a>';
                        }

                        // This is the clickable hyperlink that links to the below function. It converts data into a callable attribute "itemsel"
                    }
                }
            ]
        })
        $('.tablecontainer').on('click', 'a.errorMe', function (e) {
            e.preventDefault();
            alert("You do not have permission for this action.");
        })
        $('.tablecontainer').on('click', 'a.popup', function (e) {
            e.preventDefault();
            var text = $(this).attr('itemsel');
            $.ajax({
                url: '@Url.RouteUrl(new{ action="SelectedItem", controller="Home"})',
                data: { yeep: text },
                type: 'GET',
                dataType: 'json',
                contentType: "application/json; charset=utf-8",
                success: function (data) {
                   // alert("WIN");
                },
                error: function (xhr) {
                 //   alert("Form ERROR");
                }
            });
            OpenPopup($(this).attr('href'));
        })
        function OpenPopup(pageUrl) {
            var $pageContent = $('<div/>');
            $pageContent.load(pageUrl, function () {
                $('#popupForm', $pageContent).removeData('validator');
                $('#popupForm', $pageContent).removeData('unobtrusiveValidation');
                $.validator.unobtrusive.parse('form');
            });
            $dialog = $('<div class="popupWindow" style="overflow:auto"></div>')
                .html($pageContent)
                .dialog({
                    draggable: false,
                    autoOpen: false,
                    resizable: false,
                    model: true,
                    title: 'Popup Dialog',
                    height: 550,
                    width: 1200,
                    close: function () {
                        $dialog.dialog('destroy').remove();
                        oTable.ajax.reload();
                    }
                })

            $('.popupWindow').on('submit', '#popupForm', function (e) {
                var url = $('#popupForm')[0].action;
                $.ajax({
                    type: "POST",
                    url: url,
                    data: $('#popupForm').serialize(),
                    success: function (data) {
                        if (data.status) {
                            if (status = true) {
                                alert("Success");
                            }
                            else {
                                alert("Failed");
                            }  
                            $dialog.dialog('destroy').remove();
                            oTable.ajax.reload();
                        }
                        else {
                            if (data.Ierror == "SafeF") {
                                alert("FAILED: Safe ID not found in Database");
                            }
                            else if (data.Ierror == "SerF") {
                                alert("FAILED: Item Serial number already exists");
                            }
                            else {
                                alert("FAILED: Unknown Error");
                            }                              
                        }
                    }
                })

                e.preventDefault();
            })
            $dialog.dialog('open');
        }
    })
</script>

</body>
</html>

spark-submit --class xxx.Boot --master local[1] /user/xx/app/target/app.jar

Answer 1

我想，您应该使用jar或packages选项在命令行中指定项目（包括bigquery）的依赖项，或者制作一个包含这些依赖项的uberjar。参见此处：Spark-Submit: --packages vs --jars和此处：Is it possible to create an "uber" jar containing the project classes and the project dependencies as jars with a custom manifest file?

连接到Google BQ时通过Spark-Submit运行时出错

1 个答案: