问:
RecyleView showing all items duplicates
I am trying to display posts from a server in listView. So I used recycle-view to achieve that. Everything is working fine except that ll items are displaying twice.

I counted the total fetched items from server, and the count is 5, but adapter.getItemCount is showing 10.

After searching hours on the internet, I tried following :

```<code>@Override
public long getItemId(int position) {
    return position;
}
@Override
public int getItemViewType(int position) {
    return position;
}
</code>```

and

homeFragmentAdapter.setHasStableIds(true);

Below is my fragment...

```<code>package com.example.projectName;

import static android.content.Context.MODE_PRIVATE;
import static android.webkit.ConsoleMessage.MessageLevel.LOG;
import static com.bumptech.glide.load.resource.drawable.DrawableTransitionOptions.withCrossFade;

public class HomeFollowersFragment extends Fragment implements InfiniteScrollListener.OnLoadMoreListener, RecyclerViewItemListener {

    private static final String TAG = "HomeFollowersFragment";
    private static final String URL = "https://api.androidhive.info/json/movies_2017.json";
    private RecyclerView recyclerView;
    private ProgressBar postLoader;
    FFmpeg ffmpeg;
//    private List<Movie> movieList;
//    private HomeAdapter mAdapter;

    private List<PostList> postListGlobal = new ArrayList<>();
    List<VerticalDataModal> verticalDataModals;
    List<HorizontalDataModal> horizontalDataModals;
    private SwipeRefreshLayout swipeMore;
    private InfiniteScrollListener infiniteScrollListener;
    private HomeFragmentAdapter homeFragmentAdapter;

    SharedPreferences sharedPreferences;
    private Boolean isLoggedIn = false;
    private String email = "";
    private String token = "";
    private String userId = "";

    private Dialog customLoader;
    SkeletonScreen skeletonScreen;

    private int pastVisiblesItems, visibleItemCount, totalItemCount;
    private boolean loading = false;
    private EndlessScrollListener scrollListener;

    SharedPreferences sp;
    SharedPreferences.Editor Ed;

    public HomeFollowersFragment() {
        //super();
    }
    /**
     * @return A new instance of fragment HomeFollowersFragment.
     */
    public static HomeFollowersFragment newInstance() {
        return new HomeFollowersFragment();
    }

    @Override
    public void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
    }

    @Override
    public View onCreateView(LayoutInflater inflater, ViewGroup container,
                             Bundle savedInstanceState) {
        // Inflate the layout for this fragment
        View view = inflater.inflate(R.layout.fragment_home, container, false);
//        ((AppCompatActivity) getActivity()).getSupportActionBar().show();

        try{
            sharedPreferences = getActivity().getSharedPreferences("Login", MODE_PRIVATE);
            email = sharedPreferences.getString("email", null);
            token = sharedPreferences.getString("token", null);
            isLoggedIn = sharedPreferences.getBoolean("isLoggedIn", false);
            userId = sharedPreferences.getString("id", null);
        }catch (Exception e){
            e.printStackTrace();
            Log.d("StackError", "StackError: "+e);
        }
        sp = getActivity().getSharedPreferences("Posts", MODE_PRIVATE);
        if(!isLoggedIn || token == null || userId == null){
            Intent intent = new Intent(getActivity(), RegisterActivity.class);
            intent.putExtra("loginFrom", "profile");
            startActivity(intent);
        }

        recyclerView = view.findViewById(R.id.recycler_view);
        postLoader = view.findViewById(R.id.post_loader);
        swipeMore = view.findViewById(R.id.swipe_layout);

        homeFragmentAdapter = new HomeFragmentAdapter(postListGlobal, this, "home");

        if(sp.contains("postListGlobal"))
            skeletonScreen = Skeleton.bind(recyclerView)
                    .adapter(homeFragmentAdapter)
                    .shimmer(true)
                    .angle(20)
                    .frozen(false)
                    .duration(1200)
                    .count(10)
                    .load(R.layout.item_skelton_home_page)
                    .show(); //default count is 10
        RecyclerView.LayoutManager mLayoutManager = new GridLayoutManager(getActivity(), 2);
        StaggeredGridLayoutManager sLayoutManager = new StaggeredGridLayoutManager(2, StaggeredGridLayoutManager.VERTICAL);

        recyclerView.setLayoutManager(sLayoutManager);
        homeFragmentAdapter.setHasStableIds(true);
        recyclerView.setAdapter(homeFragmentAdapter);
        recyclerView.setNestedScrollingEnabled(false);

        customLoader = new Dialog(getActivity(), R.style.crystal_range_seek_bar);
        customLoader.setCancelable(false);
        View loaderView = getLayoutInflater().inflate(R.layout.custom_loading_layout, null);
        customLoader.getWindow().getAttributes().windowAnimations = R.style.crystal_range_seek_bar;
        customLoader.getWindow().setBackgroundDrawableResource(R.color.translucent_black);

        ImageView imageLoader = loaderView.findViewById(R.id.logo_loader);
        Glide.with(this).load(R.drawable.logo_loader).into(imageLoader);
        customLoader.setContentView(loaderView);

        if(homeFragmentAdapter.getItemCount() == 0 && !loading){
            // server fetchdata
            Log.d(TAG, "no item available..");
            postLoader.setVisibility(View.VISIBLE);
            loading = true;
            fetchStoreItems();
        }else{
            postLoader.setVisibility(View.GONE);
        }
        swipeMore.setOnRefreshListener(new SwipeRefreshLayout.OnRefreshListener() {
            @Override
            public void onRefresh() {
                Log.d(TAG, "on refresh...");
                fetchStoreItems();
            }
        });
        return view;
    }
    @Override
    public void onItemClicked(int position) {
        Log.d(TAG, "click position: "+position);
        Toast.makeText(getActivity(),postListGlobal.get(position).getTitle(),Toast.LENGTH_SHORT).show();
//        Toast.makeText(getActivity(),""+position, Toast.LENGTH_SHORT).show();

    }
    public int getLastVisibleItem(int[] lastVisibleItemPositions) {
        int maxSize = 0;
        for (int i = 0; i < lastVisibleItemPositions.length; i++) {
            if (i == 0) {
                maxSize = lastVisibleItemPositions[i];
            }
            else if (lastVisibleItemPositions[i] > maxSize) {
                maxSize = lastVisibleItemPositions[i];
            }
        }
        return maxSize;
    }
    @Override
    public void onLoadMore() {
        homeFragmentAdapter.addNullData();
        new Handler().postDelayed(new Runnable() {
            @Override
            public void run() {
                homeFragmentAdapter.removeNull();
                Toast.makeText(getContext(), "load more here...", Toast.LENGTH_LONG).show();
//                fetchStoreItems();
                swipeMore.setRefreshing(false);
            }
        }, 2000);
    }
    private void fetchStoreItems() {
        RequestQueue queue = Volley.newRequestQueue(getActivity());
        Log.d(TAG, "Post Data Followers: "+Constant.FETCH_POSTS_API);
        CacheRequest cacheRequest = new CacheRequest(0, Constant.FETCH_POSTS_API, new Response.Listener<NetworkResponse>() {
            @Override
            public void onResponse(NetworkResponse response) {
                try {
                    final String jsonString = new String(response.data,
                            HttpHeaderParser.parseCharset(response.headers));
                    if (response == null) {
                        Toast.makeText(getActivity(), "Couldn't fetch the store items! Pleas try again.", Toast.LENGTH_LONG).show();
                        loading = false;
                        return;
                    }
                    JSONObject postObj = new JSONObject(jsonString);
                    System.out.println("post full data... : " + postObj);
                    if (postObj.getBoolean("Status")) {
                        try {
                            postLoader.setVisibility(View.GONE);
                            JSONArray arrayResponse = postObj.optJSONArray("Data");
                            int dataArrLength = arrayResponse.length();
                            if(dataArrLength == 0){
                                Toast.makeText(getActivity(), "No posts available at this time, you can create yout own post by clicking on mic button", Toast.LENGTH_SHORT).show();
                            }
                            postListGlobal.clear();
                            Log.d(TAG, "Total Posts count: "+dataArrLength);
                            for(int i=0; i<dataArrLength; i++) {
                                try {
                                    JSONObject dataListObj = arrayResponse.optJSONObject(i);
                                    System.out.println("post full data... : " + dataListObj);

                                    JSONObject postDetailObj = dataListObj.optJSONObject("post_detail");
                                    JSONObject followDtatusObj = dataListObj.optJSONObject("follow_status");
                                    JSONArray postFilesArr = dataListObj.optJSONArray("post_files");
                                    JSONObject userDatasObj = postDetailObj.optJSONObject("user");

                                    String userId = userDatasObj.optString("id");
                                    String userName = userDatasObj.optString("email");
                                    String userImage = userDatasObj.optString("email");

                                    boolean followStatus = followDtatusObj.optBoolean("follow");

                                    String postId = postDetailObj.optString("id");
                                    String postTitle = postDetailObj.optString("post_title");
                                    String postDescription = postDetailObj.optString("post_description");
                                    String postCoverUrl = postDetailObj.optString("post_coverurl", "1");
                                    String postViewType = postDetailObj.optString("view_type", "1");
                                    String postAllowComment = postDetailObj.optString("allow_comments", "1");
                                    String postAllowDownload = postDetailObj.optString("allow_download", "1");
                                    String postTotalPost = postDetailObj.optString("total_post", "1");
                                    String postPostSection = postDetailObj.optString("post_section", "image");
                                    String postActiveStatus = postDetailObj.optString("is_active", "1");

                                    String postTotalViews = postDetailObj.optString("total_watched","0");
                                    String postTotalShare = postDetailObj.optString("total_share","0");
                                    String postTotalDownload = postDetailObj.optString("total_download","0");
                                    String postTotalReaction = postDetailObj.optString("total_reaction","0");

                                    String postTotalLike = postDetailObj.optString("total_like","0");
                                    String postTotalSmile = postDetailObj.optString("smile_reaction","0");
                                    String postTotalLaugh = postDetailObj.optString("laugh_reaction","0");
                                    String postTotalSad = postDetailObj.optString("sad_reaction","0");
                                    String postTotalLove = postDetailObj.optString("love_reaction","0");
                                    String postTotalShock = postDetailObj.optString("shock_reaction","0");

                                    int totalPostFiles = Integer.parseInt(postTotalPost);
                                    int postArrLength = postFilesArr.length();
                                    String postImageUrl = null;
                                    String postMusicUrl = null;

                                    String commonUrl = "http://serverName.com/";
                                    if(postArrLength >= 1){
                                        JSONObject dataFilesListObj = postFilesArr.optJSONObject(0);
//                                        System.out.println("post files full data... : " + dataFilesListObj);
                                        String postFileId = dataFilesListObj.optString("id");
                                        postImageUrl = dataFilesListObj.optString("image_file_path");
                                        postMusicUrl = dataFilesListObj.optString("music_file_path");
                                        System.out.println("post files full data... : " + dataFilesListObj);
                                    }
                                    System.out.println("post files full data... : " + commonUrl+postMusicUrl);
                                    System.out.println("post files full data... : " + commonUrl+postImageUrl);
                                    PostList postList = new PostList();
                                    postList.setId(postId);
                                    postList.setTitle(postTitle);
                                    postList.setTotalPost(""+dataArrLength);
                                    postList.setTotalView(postTotalViews);
                                    postList.setTotalReaction(postTotalReaction);
                                    postList.setMusicPath(commonUrl+postMusicUrl);

                                    postList.setImagePath(commonUrl+postImageUrl);
                                    if(postImageUrl == null){
                                        postList.setImagePath("https://amazonBucket.s3.location.amazonaws.com/images/pic1.jpg");
                                    }
                                    postList.setUserId(userId);
                                    postList.setUserName(userName);
                                    postList.setPostDataObject(arrayResponse);
                                    postListGlobal.add(postList);

                                    Log.d(TAG, "Total Posts: "+dataListObj);

                                } catch (Exception e) {
                                    e.printStackTrace();
                                    Log.d(TAG, "Post Data Error1: "+e);
                                    Toast.makeText(getActivity(), "File now found", Toast.LENGTH_LONG).show();
                                    loading = false;
                                }

                            }

                        } catch (Exception e){
                            e.printStackTrace();
                            Log.d(TAG, "Post Data Error2: "+e);
                            Toast.makeText(getActivity(), R.string.server_error, Toast.LENGTH_LONG).show();
                            loading = false;
                        }
                    }else{
                        try {
                            Toast.makeText(getActivity(), new JSONObject(jsonString).getString("Message"), Toast.LENGTH_LONG).show();
                        } catch (JSONException ex) {
                            ex.printStackTrace();
                            Log.d(TAG, "Post Data Error3: "+ex);
                            Toast.makeText(getActivity(), R.string.server_error, Toast.LENGTH_SHORT).show();
                        }
                        loading = false;
                    }

                    // refreshing recycler view
                    homeFragmentAdapter.removeNull();
                    homeFragmentAdapter.addData(postListGlobal);
                    homeFragmentAdapter.notifyDataSetChanged();
//                    save in local memory
//                    saveArrayList(postListGlobal, "postListGlobal");

                } catch (Exception e) {
                    e.printStackTrace();
                    Log.d(TAG, "Post Data Error4: "+e);
                }
                loading = true;
                homeFragmentAdapter.notifyDataSetChanged();
                swipeMore.setRefreshing(false);
            }
        }, new Response.ErrorListener() {
            @Override
            public void onErrorResponse(VolleyError error) {
                Toast.makeText(getActivity(), "onErrorResponse: "+ error, Toast.LENGTH_SHORT).show();
                swipeMore.setRefreshing(false);
                loading = true;
                homeFragmentAdapter.notifyDataSetChanged();
                postLoader.setVisibility(View.VISIBLE);
                loading = false;
                Log.d(TAG, "Post Data Error5: "+error);
            }
        }){
            @Override
            public Map<String, String> getHeaders() throws AuthFailureError {
                Map<String, String>  params = new HashMap<String, String>();
                String finalToken = "Bearer  "+token;
                params.put("Authorization", finalToken);
                params.put("Content-Type", "application/json");

                return params;
            }
        };
        // Add the request to the RequestQueue.
        queue.add(cacheRequest);

    }
    private class CacheRequest extends Request<NetworkResponse> {
        private final Response.Listener<NetworkResponse> mListener;
        private final Response.ErrorListener mErrorListener;

        public CacheRequest(int method, String url, Response.Listener<NetworkResponse> listener, Response.ErrorListener errorListener) {
            super(method, url, errorListener);
            this.mListener = listener;
            this.mErrorListener = errorListener;
        }
        @Override
        protected Response<NetworkResponse> parseNetworkResponse(NetworkResponse response) {
            Cache.Entry cacheEntry = HttpHeaderParser.parseCacheHeaders(response);
            if (cacheEntry == null) {
                cacheEntry = new Cache.Entry();
            }
            final long cacheHitButRefreshed = 3 * 60 * 1000; // in 3 minutes cache will be hit, but also refreshed on background
            final long cacheExpired = 24 * 60 * 60 * 1000; // in 24 hours this cache entry expires completely
            long now = System.currentTimeMillis();
            final long softExpire = now + cacheHitButRefreshed;
            final long ttl = now + cacheExpired;
            cacheEntry.data = response.data;
            cacheEntry.softTtl = softExpire;
            cacheEntry.ttl = ttl;
            String headerValue;
            headerValue = response.headers.get("Date");
            if (headerValue != null) {
                cacheEntry.serverDate = HttpHeaderParser.parseDateAsEpoch(headerValue);
            }
            headerValue = response.headers.get("Last-Modified");
            if (headerValue != null) {
                cacheEntry.lastModified = HttpHeaderParser.parseDateAsEpoch(headerValue);
            }
            cacheEntry.responseHeaders = response.headers;
            return Response.success(response, cacheEntry);
        }
        @Override
        protected void deliverResponse(NetworkResponse response) {
            mListener.onResponse(response);
        }
        @Override
        protected VolleyError parseNetworkError(VolleyError volleyError) {
            Log.d(TAG, "Post Data volleyError: "+volleyError);
            return super.parseNetworkError(volleyError);
        }
        @Override
        public void deliverError(VolleyError error) {
            mErrorListener.onErrorResponse(error);
        }
    }
}
</code>```

and Adapter Class

```<code>package com.example.ProjectName;

public class HomeFragmentAdapter extends RecyclerView.Adapter <HomeFragmentAdapter.HomeViewHolder>{

    //    private ArrayList<Integer> dataList;
    private List<PostList> postListGlobal;
    int VIEW_TYPE_LOADING;
    int VIEW_TYPE_ITEM;
    Context context;
    private RecyclerViewItemListener callback;
    FFmpeg ffmpeg;
    String callingPage;

    public HomeFragmentAdapter(List<PostList> postListGlobal, RecyclerViewItemListener callback, String callingPage) {
        this.postListGlobal = postListGlobal;
        this.callback = callback;
        this.callingPage = callingPage;
//        setHasStableIds(true);
    }

    @NonNull
    @Override
    public HomeFragmentAdapter.HomeViewHolder onCreateViewHolder(@NonNull ViewGroup parent, int viewType) {
        View root = null;
        context = parent.getContext();
        root = LayoutInflater.from(parent.getContext()).inflate(R.layout.adapter_home_tile_list, parent, false);
        return new DataViewHolder(root);
    }

    @Override
    public void onBindViewHolder(@NonNull HomeFragmentAdapter.HomeViewHolder holder, int position) {
        if (holder instanceof DataViewHolder) {
            final PostList postList = postListGlobal.get(position);
            holder.postTitle.setText(postList.getTitle());
            holder.postWatch.setText(postList.getTotalView());
            holder.postReaction.setText(postList.getTotalReaction());

            String imageUrl = postList.getImagePath();
//            String imageUrl = Constant.SERVER_URL+"/"+postList.getImagePath();
            String musicUrl = postList.getMusicPath();
//            String musicUrl = Constant.SERVER_URL+"/"+postList.getMusicPath();
            Log.d(TAG, "Post url: "+imageUrl+" -- "+musicUrl);
//            int totalMusicTime = getDurationVal(musicUrl, "second");
            holder.postTime.setText(postList.getTotalPost());

            holder.thumbnail.setOnClickListener(new View.OnClickListener() {
                @Override
                public void onClick(View view) {
                    callback.onItemClicked(position);
                    Log.d("homeView", "screenName : "+callingPage);
                    if(callingPage.equals("home")){
                        Log.d("homeView", "screenName : "+position);
                        Intent intent = new Intent(context, MainViewActivity.class);
                        intent.putExtra("loginFrom", "homeView");
                        intent.putExtra("postDataObj", postList.getPostDataObject().toString());
                        intent.putExtra("postPosition", ""+position);
                        intent.putExtra("tabId", "1");
                        context.startActivity(intent);
                    }
                }
            });

            Drawable mDefaultBackground = context.getResources().getDrawable(R.drawable.influencers);
            CircularProgressDrawable circularProgressDrawable = new CircularProgressDrawable(context);
            circularProgressDrawable.setStrokeWidth(5f);
            Glide.with(context)
                    .load(imageUrl)
                    .listener(new RequestListener<Drawable>() {
                        @Override
                        public boolean onLoadFailed(@Nullable GlideException e, Object model, Target<Drawable> target, boolean isFirstResource) {
//                            progressBar.setVisibility(View.GONE);
                            return false;
                        }

                        @Override
                        public boolean onResourceReady(Drawable resource, Object model, Target<Drawable> target, DataSource dataSource, boolean isFirstResource) {
//                            progressBar.setVisibility(View.GONE);
                            return false;
                        }
                    })
                    .error(mDefaultBackground)
                    .into(holder.thumbnail);
        }else{
            //Do whatever you want. Or nothing !!
        }
    }

    @Override
    public int getItemCount() {
        return postListGlobal.size();
    }

    class DataViewHolder extends HomeViewHolder {
        public DataViewHolder(View itemView) {
            super(itemView);
        }

    }

    class ProgressViewHolder extends HomeViewHolder {
        public ProgressViewHolder(View itemView) {
            super(itemView);
        }
    }

    class HomeViewHolder extends RecyclerView.ViewHolder {
        public TextView postTitle, postTime, postWatch, postReaction;
        public ImageView thumbnail;
        public HomeViewHolder(View itemView) {
            super(itemView);
            postTitle = itemView.findViewById(R.id.post_title);
            postTime = itemView.findViewById(R.id.total_time);
            postWatch = itemView.findViewById(R.id.total_watch);
            postReaction = itemView.findViewById(R.id.total_reaction);
            thumbnail = itemView.findViewById(R.id.thumbnail);
    }
    @Override
    public long getItemId(int position) {
        return position;
    }
    @Override
    public int getItemViewType(int position) {
        return position;
    }
    public void addNullData() {
    }
    public void removeNull() {
        notifyItemRemoved(postListGlobal.size());
    }
    public void addData(List<PostList> postLists) {
        postListGlobal.addAll(postLists);
        notifyDataSetChanged();
    }
}
</code>```

After trying everything, I was still not able to resolve the issue. Any help/suggestions are welcome. Let me know If I left out any needed code--if so I can update it here.

答:
postListGlobal.clear() before adding new list to the adapter.
And then notifyDataSetChanged() to let adapter know of the changes. </p>

Title: RecyleView showing all items duplicates

Body:
I am trying to display posts from a server in listView. So I used recycle-view to achieve that. Everything is working fine except that ll items are displaying twice.

I counted the total fetched items from server, and the count is 5, but adapter.getItemCount is showing 10.

After searching hours on the internet, I tried following :

@Override
public long getItemId(int position) {
    return position;
}
@Override
public int getItemViewType(int position) {
    return position;
}

and

homeFragmentAdapter.setHasStableIds(true);

Below is my fragment...

package com.example.projectName;

import static android.content.Context.MODE_PRIVATE;
import static android.webkit.ConsoleMessage.MessageLevel.LOG;
import static com.bumptech.glide.load.resource.drawable.DrawableTransitionOptions.withCrossFade;

public class HomeFollowersFragment extends Fragment implements InfiniteScrollListener.OnLoadMoreListener, RecyclerViewItemListener {

    private static final String TAG = "HomeFollowersFragment";
    private static final String URL = "https://api.androidhive.info/json/movies_2017.json";
    private RecyclerView recyclerView;
    private ProgressBar postLoader;
    FFmpeg ffmpeg;
//    private List<Movie> movieList;
//    private HomeAdapter mAdapter;

    private List<PostList> postListGlobal = new ArrayList<>();
    List<VerticalDataModal> verticalDataModals;
    List<HorizontalDataModal> horizontalDataModals;
    private SwipeRefreshLayout swipeMore;
    private InfiniteScrollListener infiniteScrollListener;
    private HomeFragmentAdapter homeFragmentAdapter;

    SharedPreferences sharedPreferences;
    private Boolean isLoggedIn = false;
    private String email = "";
    private String token = "";
    private String userId = "";

    private Dialog customLoader;
    SkeletonScreen skeletonScreen;

    private int pastVisiblesItems, visibleItemCount, totalItemCount;
    private boolean loading = false;
    private EndlessScrollListener scrollListener;

    SharedPreferences sp;
    SharedPreferences.Editor Ed;

    public HomeFollowersFragment() {
        //super();
    }
    /**
     * @return A new instance of fragment HomeFollowersFragment.
     */
    public static HomeFollowersFragment newInstance() {
        return new HomeFollowersFragment();
    }

    @Override
    public void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
    }

    @Override
    public View onCreateView(LayoutInflater inflater, ViewGroup container,
                             Bundle savedInstanceState) {
        // Inflate the layout for this fragment
        View view = inflater.inflate(R.layout.fragment_home, container, false);
//        ((AppCompatActivity) getActivity()).getSupportActionBar().show();

        try{
            sharedPreferences = getActivity().getSharedPreferences("Login", MODE_PRIVATE);
            email = sharedPreferences.getString("email", null);
            token = sharedPreferences.getString("token", null);
            isLoggedIn = sharedPreferences.getBoolean("isLoggedIn", false);
            userId = sharedPreferences.getString("id", null);
        }catch (Exception e){
            e.printStackTrace();
            Log.d("StackError", "StackError: "+e);
        }
        sp = getActivity().getSharedPreferences("Posts", MODE_PRIVATE);
        if(!isLoggedIn || token == null || userId == null){
            Intent intent = new Intent(getActivity(), RegisterActivity.class);
            intent.putExtra("loginFrom", "profile");
            startActivity(intent);
        }

        recyclerView = view.findViewById(R.id.recycler_view);
        postLoader = view.findViewById(R.id.post_loader);
        swipeMore = view.findViewById(R.id.swipe_layout);

        homeFragmentAdapter = new HomeFragmentAdapter(postListGlobal, this, "home");

        if(sp.contains("postListGlobal"))
            skeletonScreen = Skeleton.bind(recyclerView)
                    .adapter(homeFragmentAdapter)
                    .shimmer(true)
                    .angle(20)
                    .frozen(false)
                    .duration(1200)
                    .count(10)
                    .load(R.layout.item_skelton_home_page)
                    .show(); //default count is 10
        RecyclerView.LayoutManager mLayoutManager = new GridLayoutManager(getActivity(), 2);
        StaggeredGridLayoutManager sLayoutManager = new StaggeredGridLayoutManager(2, StaggeredGridLayoutManager.VERTICAL);

        recyclerView.setLayoutManager(sLayoutManager);
        homeFragmentAdapter.setHasStableIds(true);
        recyclerView.setAdapter(homeFragmentAdapter);
        recyclerView.setNestedScrollingEnabled(false);

        customLoader = new Dialog(getActivity(), R.style.crystal_range_seek_bar);
        customLoader.setCancelable(false);
        View loaderView = getLayoutInflater().inflate(R.layout.custom_loading_layout, null);
        customLoader.getWindow().getAttributes().windowAnimations = R.style.crystal_range_seek_bar;
        customLoader.getWindow().setBackgroundDrawableResource(R.color.translucent_black);

        ImageView imageLoader = loaderView.findViewById(R.id.logo_loader);
        Glide.with(this).load(R.drawable.logo_loader).into(imageLoader);
        customLoader.setContentView(loaderView);

        if(homeFragmentAdapter.getItemCount() == 0 && !loading){
            // server fetchdata
            Log.d(TAG, "no item available..");
            postLoader.setVisibility(View.VISIBLE);
            loading = true;
            fetchStoreItems();
        }else{
            postLoader.setVisibility(View.GONE);
        }
        swipeMore.setOnRefreshListener(new SwipeRefreshLayout.OnRefreshListener() {
            @Override
            public void onRefresh() {
                Log.d(TAG, "on refresh...");
                fetchStoreItems();
            }
        });
        return view;
    }
    @Override
    public void onItemClicked(int position) {
        Log.d(TAG, "click position: "+position);
        Toast.makeText(getActivity(),postListGlobal.get(position).getTitle(),Toast.LENGTH_SHORT).show();
//        Toast.makeText(getActivity(),""+position, Toast.LENGTH_SHORT).show();

    }
    public int getLastVisibleItem(int[] lastVisibleItemPositions) {
        int maxSize = 0;
        for (int i = 0; i < lastVisibleItemPositions.length; i++) {
            if (i == 0) {
                maxSize = lastVisibleItemPositions[i];
            }
            else if (lastVisibleItemPositions[i] > maxSize) {
                maxSize = lastVisibleItemPositions[i];
            }
        }
        return maxSize;
    }
    @Override
    public void onLoadMore() {
        homeFragmentAdapter.addNullData();
        new Handler().postDelayed(new Runnable() {
            @Override
            public void run() {
                homeFragmentAdapter.removeNull();
                Toast.makeText(getContext(), "load more here...", Toast.LENGTH_LONG).show();
//                fetchStoreItems();
                swipeMore.setRefreshing(false);
            }
        }, 2000);
    }
    private void fetchStoreItems() {
        RequestQueue queue = Volley.newRequestQueue(getActivity());
        Log.d(TAG, "Post Data Followers: "+Constant.FETCH_POSTS_API);
        CacheRequest cacheRequest = new CacheRequest(0, Constant.FETCH_POSTS_API, new Response.Listener<NetworkResponse>() {
            @Override
            public void onResponse(NetworkResponse response) {
                try {
                    final String jsonString = new String(response.data,
                            HttpHeaderParser.parseCharset(response.headers));
                    if (response == null) {
                        Toast.makeText(getActivity(), "Couldn't fetch the store items! Pleas try again.", Toast.LENGTH_LONG).show();
                        loading = false;
                        return;
                    }
                    JSONObject postObj = new JSONObject(jsonString);
                    System.out.println("post full data... : " + postObj);
                    if (postObj.getBoolean("Status")) {
                        try {
                            postLoader.setVisibility(View.GONE);
                            JSONArray arrayResponse = postObj.optJSONArray("Data");
                            int dataArrLength = arrayResponse.length();
                            if(dataArrLength == 0){
                                Toast.makeText(getActivity(), "No posts available at this time, you can create yout own post by clicking on mic button", Toast.LENGTH_SHORT).show();
                            }
                            postListGlobal.clear();
                            Log.d(TAG, "Total Posts count: "+dataArrLength);
                            for(int i=0; i<dataArrLength; i++) {
                                try {
                                    JSONObject dataListObj = arrayResponse.optJSONObject(i);
                                    System.out.println("post full data... : " + dataListObj);

                                    JSONObject postDetailObj = dataListObj.optJSONObject("post_detail");
                                    JSONObject followDtatusObj = dataListObj.optJSONObject("follow_status");
                                    JSONArray postFilesArr = dataListObj.optJSONArray("post_files");
                                    JSONObject userDatasObj = postDetailObj.optJSONObject("user");

                                    String userId = userDatasObj.optString("id");
                                    String userName = userDatasObj.optString("email");
                                    String userImage = userDatasObj.optString("email");

                                    boolean followStatus = followDtatusObj.optBoolean("follow");

                                    String postId = postDetailObj.optString("id");
                                    String postTitle = postDetailObj.optString("post_title");
                                    String postDescription = postDetailObj.optString("post_description");
                                    String postCoverUrl = postDetailObj.optString("post_coverurl", "1");
                                    String postViewType = postDetailObj.optString("view_type", "1");
                                    String postAllowComment = postDetailObj.optString("allow_comments", "1");
                                    String postAllowDownload = postDetailObj.optString("allow_download", "1");
                                    String postTotalPost = postDetailObj.optString("total_post", "1");
                                    String postPostSection = postDetailObj.optString("post_section", "image");
                                    String postActiveStatus = postDetailObj.optString("is_active", "1");

                                    String postTotalViews = postDetailObj.optString("total_watched","0");
                                    String postTotalShare = postDetailObj.optString("total_share","0");
                                    String postTotalDownload = postDetailObj.optString("total_download","0");
                                    String postTotalReaction = postDetailObj.optString("total_reaction","0");

                                    String postTotalLike = postDetailObj.optString("total_like","0");
                                    String postTotalSmile = postDetailObj.optString("smile_reaction","0");
                                    String postTotalLaugh = postDetailObj.optString("laugh_reaction","0");
                                    String postTotalSad = postDetailObj.optString("sad_reaction","0");
                                    String postTotalLove = postDetailObj.optString("love_reaction","0");
                                    String postTotalShock = postDetailObj.optString("shock_reaction","0");

                                    int totalPostFiles = Integer.parseInt(postTotalPost);
                                    int postArrLength = postFilesArr.length();
                                    String postImageUrl = null;
                                    String postMusicUrl = null;

                                    String commonUrl = "http://serverName.com/";
                                    if(postArrLength >= 1){
                                        JSONObject dataFilesListObj = postFilesArr.optJSONObject(0);
//                                        System.out.println("post files full data... : " + dataFilesListObj);
                                        String postFileId = dataFilesListObj.optString("id");
                                        postImageUrl = dataFilesListObj.optString("image_file_path");
                                        postMusicUrl = dataFilesListObj.optString("music_file_path");
                                        System.out.println("post files full data... : " + dataFilesListObj);
                                    }
                                    System.out.println("post files full data... : " + commonUrl+postMusicUrl);
                                    System.out.println("post files full data... : " + commonUrl+postImageUrl);
                                    PostList postList = new PostList();
                                    postList.setId(postId);
                                    postList.setTitle(postTitle);
                                    postList.setTotalPost(""+dataArrLength);
                                    postList.setTotalView(postTotalViews);
                                    postList.setTotalReaction(postTotalReaction);
                                    postList.setMusicPath(commonUrl+postMusicUrl);

                                    postList.setImagePath(commonUrl+postImageUrl);
                                    if(postImageUrl == null){
                                        postList.setImagePath("https://amazonBucket.s3.location.amazonaws.com/images/pic1.jpg");
                                    }
                                    postList.setUserId(userId);
                                    postList.setUserName(userName);
                                    postList.setPostDataObject(arrayResponse);
                                    postListGlobal.add(postList);

                                    Log.d(TAG, "Total Posts: "+dataListObj);

                                } catch (Exception e) {
                                    e.printStackTrace();
                                    Log.d(TAG, "Post Data Error1: "+e);
                                    Toast.makeText(getActivity(), "File now found", Toast.LENGTH_LONG).show();
                                    loading = false;
                                }

                            }

                        } catch (Exception e){
                            e.printStackTrace();
                            Log.d(TAG, "Post Data Error2: "+e);
                            Toast.makeText(getActivity(), R.string.server_error, Toast.LENGTH_LONG).show();
                            loading = false;
                        }
                    }else{
                        try {
                            Toast.makeText(getActivity(), new JSONObject(jsonString).getString("Message"), Toast.LENGTH_LONG).show();
                        } catch (JSONException ex) {
                            ex.printStackTrace();
                            Log.d(TAG, "Post Data Error3: "+ex);
                            Toast.makeText(getActivity(), R.string.server_error, Toast.LENGTH_SHORT).show();
                        }
                        loading = false;
                    }

                    // refreshing recycler view
                    homeFragmentAdapter.removeNull();
                    homeFragmentAdapter.addData(postListGlobal);
                    homeFragmentAdapter.notifyDataSetChanged();
//                    save in local memory
//                    saveArrayList(postListGlobal, "postListGlobal");

                } catch (Exception e) {
                    e.printStackTrace();
                    Log.d(TAG, "Post Data Error4: "+e);
                }
                loading = true;
                homeFragmentAdapter.notifyDataSetChanged();
                swipeMore.setRefreshing(false);
            }
        }, new Response.ErrorListener() {
            @Override
            public void onErrorResponse(VolleyError error) {
                Toast.makeText(getActivity(), "onErrorResponse: "+ error, Toast.LENGTH_SHORT).show();
                swipeMore.setRefreshing(false);
                loading = true;
                homeFragmentAdapter.notifyDataSetChanged();
                postLoader.setVisibility(View.VISIBLE);
                loading = false;
                Log.d(TAG, "Post Data Error5: "+error);
            }
        }){
            @Override
            public Map<String, String> getHeaders() throws AuthFailureError {
                Map<String, String>  params = new HashMap<String, String>();
                String finalToken = "Bearer  "+token;
                params.put("Authorization", finalToken);
                params.put("Content-Type", "application/json");

                return params;
            }
        };
        // Add the request to the RequestQueue.
        queue.add(cacheRequest);

    }
    private class CacheRequest extends Request<NetworkResponse> {
        private final Response.Listener<NetworkResponse> mListener;
        private final Response.ErrorListener mErrorListener;

        public CacheRequest(int method, String url, Response.Listener<NetworkResponse> listener, Response.ErrorListener errorListener) {
            super(method, url, errorListener);
            this.mListener = listener;
            this.mErrorListener = errorListener;
        }
        @Override
        protected Response<NetworkResponse> parseNetworkResponse(NetworkResponse response) {
            Cache.Entry cacheEntry = HttpHeaderParser.parseCacheHeaders(response);
            if (cacheEntry == null) {
                cacheEntry = new Cache.Entry();
            }
            final long cacheHitButRefreshed = 3 * 60 * 1000; // in 3 minutes cache will be hit, but also refreshed on background
            final long cacheExpired = 24 * 60 * 60 * 1000; // in 24 hours this cache entry expires completely
            long now = System.currentTimeMillis();
            final long softExpire = now + cacheHitButRefreshed;
            final long ttl = now + cacheExpired;
            cacheEntry.data = response.data;
            cacheEntry.softTtl = softExpire;
            cacheEntry.ttl = ttl;
            String headerValue;
            headerValue = response.headers.get("Date");
            if (headerValue != null) {
                cacheEntry.serverDate = HttpHeaderParser.parseDateAsEpoch(headerValue);
            }
            headerValue = response.headers.get("Last-Modified");
            if (headerValue != null) {
                cacheEntry.lastModified = HttpHeaderParser.parseDateAsEpoch(headerValue);
            }
            cacheEntry.responseHeaders = response.headers;
            return Response.success(response, cacheEntry);
        }
        @Override
        protected void deliverResponse(NetworkResponse response) {
            mListener.onResponse(response);
        }
        @Override
        protected VolleyError parseNetworkError(VolleyError volleyError) {
            Log.d(TAG, "Post Data volleyError: "+volleyError);
            return super.parseNetworkError(volleyError);
        }
        @Override
        public void deliverError(VolleyError error) {
            mErrorListener.onErrorResponse(error);
        }
    }
}

and Adapter Class

package com.example.ProjectName;

public class HomeFragmentAdapter extends RecyclerView.Adapter <HomeFragmentAdapter.HomeViewHolder>{

    //    private ArrayList<Integer> dataList;
    private List<PostList> postListGlobal;
    int VIEW_TYPE_LOADING;
    int VIEW_TYPE_ITEM;
    Context context;
    private RecyclerViewItemListener callback;
    FFmpeg ffmpeg;
    String callingPage;

    public HomeFragmentAdapter(List<PostList> postListGlobal, RecyclerViewItemListener callback, String callingPage) {
        this.postListGlobal = postListGlobal;
        this.callback = callback;
        this.callingPage = callingPage;
//        setHasStableIds(true);
    }

    @NonNull
    @Override
    public HomeFragmentAdapter.HomeViewHolder onCreateViewHolder(@NonNull ViewGroup parent, int viewType) {
        View root = null;
        context = parent.getContext();
        root = LayoutInflater.from(parent.getContext()).inflate(R.layout.adapter_home_tile_list, parent, false);
        return new DataViewHolder(root);
    }

    @Override
    public void onBindViewHolder(@NonNull HomeFragmentAdapter.HomeViewHolder holder, int position) {
        if (holder instanceof DataViewHolder) {
            final PostList postList = postListGlobal.get(position);
            holder.postTitle.setText(postList.getTitle());
            holder.postWatch.setText(postList.getTotalView());
            holder.postReaction.setText(postList.getTotalReaction());

            String imageUrl = postList.getImagePath();
//            String imageUrl = Constant.SERVER_URL+"/"+postList.getImagePath();
            String musicUrl = postList.getMusicPath();
//            String musicUrl = Constant.SERVER_URL+"/"+postList.getMusicPath();
            Log.d(TAG, "Post url: "+imageUrl+" -- "+musicUrl);
//            int totalMusicTime = getDurationVal(musicUrl, "second");
            holder.postTime.setText(postList.getTotalPost());

            holder.thumbnail.setOnClickListener(new View.OnClickListener() {
                @Override
                public void onClick(View view) {
                    callback.onItemClicked(position);
                    Log.d("homeView", "screenName : "+callingPage);
                    if(callingPage.equals("home")){
                        Log.d("homeView", "screenName : "+position);
                        Intent intent = new Intent(context, MainViewActivity.class);
                        intent.putExtra("loginFrom", "homeView");
                        intent.putExtra("postDataObj", postList.getPostDataObject().toString());
                        intent.putExtra("postPosition", ""+position);
                        intent.putExtra("tabId", "1");
                        context.startActivity(intent);
                    }
                }
            });

            Drawable mDefaultBackground = context.getResources().getDrawable(R.drawable.influencers);
            CircularProgressDrawable circularProgressDrawable = new CircularProgressDrawable(context);
            circularProgressDrawable.setStrokeWidth(5f);
            Glide.with(context)
                    .load(imageUrl)
                    .listener(new RequestListener<Drawable>() {
                        @Override
                        public boolean onLoadFailed(@Nullable GlideException e, Object model, Target<Drawable> target, boolean isFirstResource) {
//                            progressBar.setVisibility(View.GONE);
                            return false;
                        }

                        @Override
                        public boolean onResourceReady(Drawable resource, Object model, Target<Drawable> target, DataSource dataSource, boolean isFirstResource) {
//                            progressBar.setVisibility(View.GONE);
                            return false;
                        }
                    })
                    .error(mDefaultBackground)
                    .into(holder.thumbnail);
        }else{
            //Do whatever you want. Or nothing !!
        }
    }

    @Override
    public int getItemCount() {
        return postListGlobal.size();
    }

    class DataViewHolder extends HomeViewHolder {
        public DataViewHolder(View itemView) {
            super(itemView);
        }

    }

    class ProgressViewHolder extends HomeViewHolder {
        public ProgressViewHolder(View itemView) {
            super(itemView);
        }
    }

    class HomeViewHolder extends RecyclerView.ViewHolder {
        public TextView postTitle, postTime, postWatch, postReaction;
        public ImageView thumbnail;
        public HomeViewHolder(View itemView) {
            super(itemView);
            postTitle = itemView.findViewById(R.id.post_title);
            postTime = itemView.findViewById(R.id.total_time);
            postWatch = itemView.findViewById(R.id.total_watch);
            postReaction = itemView.findViewById(R.id.total_reaction);
            thumbnail = itemView.findViewById(R.id.thumbnail);
    }
    @Override
    public long getItemId(int position) {
        return position;
    }
    @Override
    public int getItemViewType(int position) {
        return position;
    }
    public void addNullData() {
    }
    public void removeNull() {
        notifyItemRemoved(postListGlobal.size());
    }
    public void addData(List<PostList> postLists) {
        postListGlobal.addAll(postLists);
        notifyDataSetChanged();
    }
}

After trying everything, I was still not able to resolve the issue. Any help/suggestions are welcome. Let me know If I left out any needed code--if so I can update it here.


来源: stackexchange

问:
How to read from a high IO dataset in pytorch which grows from epoch to epoch
I use Tensorflow, but I'm writing documentation for users that will typically vary across deep learning frameworks.

When working with datasets that don't fit on the local filesystem (TB+) I sample data from a remote data store and write samples locally to a Tensorflow standardtfrecords format.

During the first epoch of training I will have only sampled a few values, therefore an epoch of local data is very small, I train on it. On epoch 2 I re-examine what data files have been produced by my sampling subprocesses (now more) and train on the expanded set of local data files for the next epoch. Repeat the process each epoch. In this way I build up a local cache of samples and can evict older samples as I fill up the local storage. The local samples cache grows at about the time the model needs the variance the most (towards the latter part of training).

In Python/Tensorflow it's crucial that I not deserialize the data in the Python training loop process because the Python GIL can't support the data transfer rates (300-600 MB/sec, the data is raw scientific uncompressible), and thus GPU performance suffers when the Python GIL can't service the training loop fast.

Writing the samples to a tfrecords file from subprocesses (python multiprocessing) allows tensorflow's native TFRecordsDataset to do deserialization outside of Python and thus we sidestep the Python GIL issues, and I can saturate a GPU with high IO data rates.

I would like to know how I would address this issue in Pytorch. I'm writing about the sampling strategy that's being used, and want to provide specific recommendations to users of both Tensorflow and PyTorch, but I don't know the PyTorch preprocessing ecosystem well enough to write with sufficient detail.

Side note: the only purely Python based solution to support these data transfer rates may come in Python 3.8 with System V shared memory and multiprocessing, but I haven't tried that yet as support for it isn't quite sufficient (soon it will be). Existing multiprocessing solutions aren't sufficient because they require deserialization in the training loop process and thus lock the GIL during deserialization at high IO rates.

答:
Actually, you can easily deserialize data in a subprocess by using torch.utils.data.DataLoader. By setting num_workers argument to 1 or a bigger value, you can spawn subprocesses with their own python interpreters and GILs.

```<code>loader = torch.utils.data.DataLoader(your_dataset, num_workers=n, **kwargs)
for epoch in range(epochs):
    for batch_idx, data in enumerate(loader):
         # loader in the main process does not claim GIL at this point
</code>```

A Dataloader requires a torch.utils.data.Dataset to get data from. It may not be a trivial job to implement a proper subclass in your case. In case you need to recreate a Dataset instance for every epoch, you can do something like this.

```<code>for epcoh in range(epochs):
    dset = get_new_dataset()
    loader = torch.utils.data.DataLoader(dset, num_workers=n, **kwargs)
    for batch_idx, data in enumerate(loader):
        # Do training
</code>```

or even better

```<code>dset = get_new_dataset()
loader = torch.utils.data.DataLoader(dset, num_workers=n, **kwargs)

for epcoh in range(epochs):
    last_batch_idx =  (len(dset)-1) // loader.batch_size
    for batch_idx, data in enumerate(loader):
        # Prepare next loader in advance to avoid blocking
        if batch_idx == last_batch_idx:
            dset = get_new_dataset()
            loader = torch.utils.data.DataLoader(dset, num_workers=n, **kwargs)
        # Do training
</code>```

As a side note, please note that it's CPU bound operation that is affected by GIL in most cases, not I/O bound operation, i.e., threading will do for any purely I/O heavy operation and you don't even need subprocess. For more information please refer to this question and this wikipedia article.

ContentLicense: CC BY-SA 4.0

Body:
Actually, you can easily deserialize data in a subprocess by using torch.utils.data.DataLoader. By setting num_workers argument to 1 or a bigger value, you can spawn subprocesses with their own python interpreters and GILs.

loader = torch.utils.data.DataLoader(your_dataset, num_workers=n, **kwargs)
for epoch in range(epochs):
    for batch_idx, data in enumerate(loader):
         # loader in the main process does not claim GIL at this point

A Dataloader requires a torch.utils.data.Dataset to get data from. It may not be a trivial job to implement a proper subclass in your case. In case you need to recreate a Dataset instance for every epoch, you can do something like this.

for epcoh in range(epochs):
    dset = get_new_dataset()
    loader = torch.utils.data.DataLoader(dset, num_workers=n, **kwargs)
    for batch_idx, data in enumerate(loader):
        # Do training

or even better

dset = get_new_dataset()
loader = torch.utils.data.DataLoader(dset, num_workers=n, **kwargs)

for epcoh in range(epochs):
    last_batch_idx =  (len(dset)-1) // loader.batch_size
    for batch_idx, data in enumerate(loader):
        # Prepare next loader in advance to avoid blocking
        if batch_idx == last_batch_idx:
            dset = get_new_dataset()
            loader = torch.utils.data.DataLoader(dset, num_workers=n, **kwargs)
        # Do training

As a side note, please note that it's CPU bound operation that is affected by GIL in most cases, not I/O bound operation, i.e., threading will do for any purely I/O heavy operation and you don't even need subprocess. For more information please refer to this question and this wikipedia article.


问:
Excel Open XML error: "found unreadable content" when creating simple example
I am getting the ambiguous "excel found unreadable content" error when I try to open the document created by the following code:

```<code>public void GenerateWorkbookFromDB()
{
    //Make a copy of the template file
    File.Copy(HttpContext.Current.Server.MapPath("ReportTemplate/test.xlsx"), HttpContext.Current.Server.MapPath("Reports/test.xlsx"), true);

    //Open up the copied template workbook
    using (SpreadsheetDocument myWorkbook = SpreadsheetDocument.Open(HttpContext.Current.Server.MapPath("Reports/test.xlsx"), true))
    {
        WorkbookPart workbookPart = myWorkbook.WorkbookPart;
        WorksheetPart worksheetPart = workbookPart.WorksheetParts.First();
        string origninalSheetId = workbookPart.GetIdOfPart(worksheetPart);

        WorksheetPart replacementPart = workbookPart.AddNewPart<WorksheetPart>();
        string replacementPartId = workbookPart.GetIdOfPart(replacementPart);

        OpenXmlReader reader = OpenXmlReader.Create(worksheetPart);
        OpenXmlWriter writer = OpenXmlWriter.Create(replacementPart);

        Row r = new Row();
        Cell c = new Cell();
        CellValue v = new CellValue();
        v.Text = "test";
        c.Append(v);

        while (reader.Read())
        {
            if (reader.ElementType == typeof(SheetData))
            {
                if (reader.IsEndElement)
                    continue;
                writer.WriteStartElement(new SheetData());

                for (int row = 0; row < 20; row++)
                {
                    writer.WriteStartElement(r);

                    for (int col = 0; col < 4; col++)
                    {
                        writer.WriteElement(c);
                    }

                    writer.WriteEndElement();
                }

                writer.WriteEndElement();
            }
            else
            {
                if (reader.IsStartElement)
                    writer.WriteStartElement(reader);
                else if (reader.IsEndElement)
                    writer.WriteEndElement();
            }
        }
        reader.Close();
        writer.Close();

        try
        {
            Sheet sheet = workbookPart.Workbook.Descendants<Sheet>().Where(s => s.Id.Value.Equals(origninalSheetId)).First();
            sheet.Id.Value = replacementPartId;
            workbookPart.DeletePart(worksheetPart);
        }
        catch (Exception ex) { }
    }
}
</code>```

any help or suggestions is much appreciated! :D

答:
I actually found a way to fix the error by changing the way I input the text into the cell itself.  Notice in the code below where I commented out the 2 lines and what I replaced them with.

```<code>public void GenerateWorkbookFromDB()
{
    //Make a copy of the template file
    File.Copy(HttpContext.Current.Server.MapPath("ReportTemplate/test.xlsx"), HttpContext.Current.Server.MapPath("Reports/test.xlsx"), true);

    //Open up the copied template workbook
    using (SpreadsheetDocument myWorkbook = SpreadsheetDocument.Open(HttpContext.Current.Server.MapPath("Reports/test.xlsx"), true))
    {
        WorkbookPart workbookPart = myWorkbook.WorkbookPart;
        WorksheetPart worksheetPart = workbookPart.WorksheetParts.First();
        string origninalSheetId = workbookPart.GetIdOfPart(worksheetPart);

        WorksheetPart replacementPart = workbookPart.AddNewPart<WorksheetPart>();
        string replacementPartId = workbookPart.GetIdOfPart(replacementPart);

        OpenXmlReader reader = OpenXmlReader.Create(worksheetPart);
        OpenXmlWriter writer = OpenXmlWriter.Create(replacementPart);

        Row r = new Row();
        Cell c = new Cell();
        string txt = "test";
        c.CellValue = new CellValue(txt.ToString());
        c.DataType = new EnumValue<CellValues>(CellValues.String);
        //v.Text = "test";
        //c.Append(v);

        while (reader.Read())
        {
            if (reader.ElementType == typeof(SheetData))
            {
                if (reader.IsEndElement)
                    continue;
                writer.WriteStartElement(new SheetData());

                for (int row = 0; row < 20; row++)
                {
                    writer.WriteStartElement(r);

                    for (int col = 0; col < 4; col++)
                    {
                        writer.WriteElement(c);
                    }

                    writer.WriteEndElement();
                }

                writer.WriteEndElement();
            }
            else
            {
                if (reader.IsStartElement)
                    writer.WriteStartElement(reader);
                else if (reader.IsEndElement)
                    writer.WriteEndElement();
            }
        }
        reader.Close();
        writer.Close();

        try
        {
            Sheet sheet = workbookPart.Workbook.Descendants<Sheet>().Where(s => s.Id.Value.Equals(origninalSheetId)).First();
            sheet.Id.Value = replacementPartId;
            workbookPart.DeletePart(worksheetPart);
        }
        catch (Exception ex) { }
    }
}
</code>```

I hope this helps anyone else who might be experiencing the same issue or something similar.

Thanks to those who tried to answer ;-)

ContentLicense: CC BY-SA 3.0

Body:
I am getting the ambiguous "excel found unreadable content" error when I try to open the document created by the following code:

public void GenerateWorkbookFromDB()
{
    //Make a copy of the template file
    File.Copy(HttpContext.Current.Server.MapPath("ReportTemplate/test.xlsx"), HttpContext.Current.Server.MapPath("Reports/test.xlsx"), true);

    //Open up the copied template workbook
    using (SpreadsheetDocument myWorkbook = SpreadsheetDocument.Open(HttpContext.Current.Server.MapPath("Reports/test.xlsx"), true))
    {
        WorkbookPart workbookPart = myWorkbook.WorkbookPart;
        WorksheetPart worksheetPart = workbookPart.WorksheetParts.First();
        string origninalSheetId = workbookPart.GetIdOfPart(worksheetPart);

        WorksheetPart replacementPart = workbookPart.AddNewPart<WorksheetPart>();
        string replacementPartId = workbookPart.GetIdOfPart(replacementPart);

        OpenXmlReader reader = OpenXmlReader.Create(worksheetPart);
        OpenXmlWriter writer = OpenXmlWriter.Create(replacementPart);

        Row r = new Row();
        Cell c = new Cell();
        CellValue v = new CellValue();
        v.Text = "test";
        c.Append(v);

        while (reader.Read())
        {
            if (reader.ElementType == typeof(SheetData))
            {
                if (reader.IsEndElement)
                    continue;
                writer.WriteStartElement(new SheetData());

                for (int row = 0; row < 20; row++)
                {
                    writer.WriteStartElement(r);

                    for (int col = 0; col < 4; col++)
                    {
                        writer.WriteElement(c);
                    }

                    writer.WriteEndElement();
                }

                writer.WriteEndElement();
            }
            else
            {
                if (reader.IsStartElement)
                    writer.WriteStartElement(reader);
                else if (reader.IsEndElement)
                    writer.WriteEndElement();
            }
        }
        reader.Close();
        writer.Close();

        try
        {
            Sheet sheet = workbookPart.Workbook.Descendants<Sheet>().Where(s => s.Id.Value.Equals(origninalSheetId)).First();
            sheet.Id.Value = replacementPartId;
            workbookPart.DeletePart(worksheetPart);
        }
        catch (Exception ex) { }
    }
}

any help or suggestions is much appreciated! :D

Body:
I actually found a way to fix the error by changing the way I input the text into the cell itself.  Notice in the code below where I commented out the 2 lines and what I replaced them with.

public void GenerateWorkbookFromDB()
{
    //Make a copy of the template file
    File.Copy(HttpContext.Current.Server.MapPath("ReportTemplate/test.xlsx"), HttpContext.Current.Server.MapPath("Reports/test.xlsx"), true);

    //Open up the copied template workbook
    using (SpreadsheetDocument myWorkbook = SpreadsheetDocument.Open(HttpContext.Current.Server.MapPath("Reports/test.xlsx"), true))
    {
        WorkbookPart workbookPart = myWorkbook.WorkbookPart;
        WorksheetPart worksheetPart = workbookPart.WorksheetParts.First();
        string origninalSheetId = workbookPart.GetIdOfPart(worksheetPart);

        WorksheetPart replacementPart = workbookPart.AddNewPart<WorksheetPart>();
        string replacementPartId = workbookPart.GetIdOfPart(replacementPart);

        OpenXmlReader reader = OpenXmlReader.Create(worksheetPart);
        OpenXmlWriter writer = OpenXmlWriter.Create(replacementPart);

        Row r = new Row();
        Cell c = new Cell();
        string txt = "test";
        c.CellValue = new CellValue(txt.ToString());
        c.DataType = new EnumValue<CellValues>(CellValues.String);
        //v.Text = "test";
        //c.Append(v);

        while (reader.Read())
        {
            if (reader.ElementType == typeof(SheetData))
            {
                if (reader.IsEndElement)
                    continue;
                writer.WriteStartElement(new SheetData());

                for (int row = 0; row < 20; row++)
                {
                    writer.WriteStartElement(r);

                    for (int col = 0; col < 4; col++)
                    {
                        writer.WriteElement(c);
                    }

                    writer.WriteEndElement();
                }

                writer.WriteEndElement();
            }
            else
            {
                if (reader.IsStartElement)
                    writer.WriteStartElement(reader);
                else if (reader.IsEndElement)
                    writer.WriteEndElement();
            }
        }
        reader.Close();
        writer.Close();

        try
        {
            Sheet sheet = workbookPart.Workbook.Descendants<Sheet>().Where(s => s.Id.Value.Equals(origninalSheetId)).First();
            sheet.Id.Value = replacementPartId;
            workbookPart.DeletePart(worksheetPart);
        }
        catch (Exception ex) { }
    }
}

I hope this helps anyone else who might be experiencing the same issue or something similar.

Thanks to those who tried to answer ;-)


问:
How to extract values from XML in PostgreSQL PL/pgSQL?
I have input varchar parameter in my PostgreSQL function:

<File xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" FILE_CREATE="2020-02-17T17:43:55" FILE_NAME="test02.txt"></File>
```

How to extract 'test02.txt' value from this XML?

答:
You can use xpath() for that:

```<code>select (xpath('/File/@FILE_NAME', parameter::xml))[1]::text
</code>```

xpath() returns an array of all matches and the [1] picks the first element from that array.

or inside PL/pgSQL:

```<code>the_variable := (xpath('/File/@FILE_NAME', parameter::xml))[1]::text;
</code>```

(Obviously you have to replace the_variable and parameter with the names that you use)

Body:
I have input varchar parameter in my PostgreSQL function:

<File xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" FILE_CREATE="2020-02-17T17:43:55" FILE_NAME="test02.txt"></File>

How to extract 'test02.txt' value from this XML?

Body:
You can use xpath() for that:

select (xpath('/File/@FILE_NAME', parameter::xml))[1]::text

xpath() returns an array of all matches and the [1] picks the first element from that array.

or inside PL/pgSQL:

the_variable := (xpath('/File/@FILE_NAME', parameter::xml))[1]::text;

(Obviously you have to replace the_variable and parameter with the names that you use)


问:
Laravel with Arabic Slugs
i'm using Laravel 6 to build a blog application
and the language of this app is Arabic
but there is a problem with slug ,it looks like that Laravel doesn't support Arabic.

any idea to fix this?

Controller

```<code>public function post($slug)
{
    $post = Post::where('slug',$slug)->first();
    return view('content.post',compact('post'));
}
</code>```

Store Method

```<code>public function store(Request $request)
{
    $this->validate($request, array(
        'title'         => 'required|max:255',
        'slug'          => 'required|min:3|max:255|unique:posts',
        'body'          => 'required',
    ));
    $post = new Post;
    $post->title = $request->input('title');
    $post->slug = Str::slug($request->slug, '-');
    $post->body = $request->input('body');
    $post->save();

    return redirect('admin/posts')->with('success', 'post is successfully saved');
}
</code>```

Route

```<code>Route::get('post/{slug}', 'PagesController@post')->name('post.show');
</code>```

答:
Laravel's slugging support by transliterating languages such as Arabic is not very good. You could alternatively use PHP's Transliterator https://www.php.net/manual/en/transliterator.transliterate.php to get the transliterated string before sending through Laravel's Str::slug function.

For example, you could use something like this:

```<code>Str::slug(transliterator_transliterate("Any-Latin; Latin-ASCII; NFD; [:Nonspacing Mark:] Remove; NFC; [:Punctuation:] Remove; Lower();", "arabic sentence you need slugged"));
</code>```

Do note that you'd need PECL intl installed for this to work.

Title: Laravel with Arabic Slugs

Body:
i'm using Laravel 6 to build a blog application
and the language of this app is Arabic
but there is a problem with slug ,it looks like that Laravel doesn't support Arabic.

any idea to fix this?

Controller

public function post($slug)
{
    $post = Post::where('slug',$slug)->first();
    return view('content.post',compact('post'));
}

Store Method

public function store(Request $request)
{
    $this->validate($request, array(
        'title'         => 'required|max:255',
        'slug'          => 'required|min:3|max:255|unique:posts',
        'body'          => 'required',
    ));
    $post = new Post;
    $post->title = $request->input('title');
    $post->slug = Str::slug($request->slug, '-');
    $post->body = $request->input('body');
    $post->save();

    return redirect('admin/posts')->with('success', 'post is successfully saved');
}

Route

Route::get('post/{slug}', 'PagesController@post')->name('post.show');

Body:
Laravel's slugging support by transliterating languages such as Arabic is not very good. You could alternatively use PHP's Transliterator https://www.php.net/manual/en/transliterator.transliterate.php to get the transliterated string before sending through Laravel's Str::slug function.

For example, you could use something like this:

Str::slug(transliterator_transliterate("Any-Latin; Latin-ASCII; NFD; [:Nonspacing Mark:] Remove; NFC; [:Punctuation:] Remove; Lower();", "arabic sentence you need slugged"));

Do note that you'd need PECL intl installed for this to work.


问:
basic query very slow on azure managed instance sql server database
On an Azure managed sql server instance database,
I do

```<code>Select top 1000 * from _CSP_Detail.
</code>```

The data table is empty because I deleted all data from the table yesterday (Delete * from _CSP_Detail).

It used to have a million rows, and it took a while to delete.  There are a hand full of indexes / indices defined on the table.

After doing the delete, today, I am finding that

```<code>Select top 1000 * from _CSP_Detail
</code>```

does pull nothing, as expected,
but takes over a minute to run.

Puzzled.
Any ideas?

答:
It seems your table is a heap (no clustered index). Space is not reclaimed when you delete rows from a heap so, even though the table is empty, the allocated pages must be scanned by the SELECT query to find rows.

Explicitly TRUNCATE the table or create/drop a clustered index to reclaim space and improve performance against the empty table.

Note that tables should generally have a clustered index according to the documentation.

Body:
On an Azure managed sql server instance database,
I do

Select top 1000 * from _CSP_Detail.

The data table is empty because I deleted all data from the table yesterday (Delete * from _CSP_Detail).

It used to have a million rows, and it took a while to delete.  There are a hand full of indexes / indices defined on the table.

After doing the delete, today, I am finding that

Select top 1000 * from _CSP_Detail

does pull nothing, as expected,
but takes over a minute to run.

Puzzled.
Any ideas?


问:
Spring Data Elastic - Java.Time.Instant class Jackson deserliization not working
Following is my WorkroomDTO:

```<code>  @NotNull
  private Instant createdOn;

  @JsonInclude(JsonInclude.Include.NON_NULL)
  private Instant changedOn;
</code>```

As you can see i am using Java 8 Instant class.
In the elasticsearch Index server i store the following as JSON:

```<code>  "createdOn": {
    "nano": 877000000,
    "epochSecond": 1579861613
  },
  "changedOn": {
    "nano": 920000000,
    "epochSecond": 1579861613
  },
</code>```

The problem is when i query the elasticsearch server to get me the workroom

```<code>    return elasticsearchOperations.queryForPage(new NativeSearchQueryBuilder().withQuery(mainQuery)
                                                                          .withPageable(elasticUtils.interceptPageable(searchDto.getPageable(), "name"))
                                                                          .build(),
                                            WorkroomDTO.class);
</code>```

, i make a mapping of these fields to my WorkroomDTO i get the following exception:

```<code>    com.fasterxml.jackson.databind.exc.InvalidDefinitionException: Cannot construct instance of `java.time.Instant` (no Creators, like default construct, exist): cannot deserialize from Object value (no delegate- or property-based Creator)
 at [Source: (String)"{"createdOn":{"nano":68000000,"epochSecond":1580127683}
</code>```

FYI:

I have created a configuration file where is register explicitly the JavaTimeModule to the Object Mapper

```<code> @Configuration
public class JacksonConfiguration {

  @Value("${application.serialization.include-type-key:true}")
  private boolean includeTypeKey = true;

  @Bean
  public ObjectMapper objectMapper() {
    ObjectMapper mapper = new ObjectMapper();
    mapper.registerModule(new JavaTimeModule());
    mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
    mapper.addHandler(new MissingTypeIdHandler());
    if (includeTypeKey) {
      mapper.setMixInResolver(new TypeKeyMixInResolver());
    }
    return mapper;
  }

}
</code>```

Need help!

答:
Well if I'm not completely wrong, your mapping fails due to the wrong format. The json you get looks like this:

```<code> "createdOn": {
    "nano": 877000000,
    "epochSecond": 1579861613
  },
  "changedOn": {
    "nano": 920000000,
    "epochSecond": 1579861613
  },
</code>```

That means you have 2 objects 'createdOn' and 'changedOn' with two properties (nano, epochSecond), while you try to map it to one object containing two properties named 'createdOn' and 'changedOn'. You need to modify that, you have e.g. a class called Entry, with two properties (nano, epochSeconds) and then a class with two properties (createdOn, changedOn) of type Entry

Body:
Following is my WorkroomDTO:

  @NotNull
  private Instant createdOn;

  @JsonInclude(JsonInclude.Include.NON_NULL)
  private Instant changedOn;

As you can see i am using Java 8 Instant class.
In the elasticsearch Index server i store the following as JSON:

  "createdOn": {
    "nano": 877000000,
    "epochSecond": 1579861613
  },
  "changedOn": {
    "nano": 920000000,
    "epochSecond": 1579861613
  },

The problem is when i query the elasticsearch server to get me the workroom

    return elasticsearchOperations.queryForPage(new NativeSearchQueryBuilder().withQuery(mainQuery)
                                                                          .withPageable(elasticUtils.interceptPageable(searchDto.getPageable(), "name"))
                                                                          .build(),
                                            WorkroomDTO.class);

, i make a mapping of these fields to my WorkroomDTO i get the following exception:

    com.fasterxml.jackson.databind.exc.InvalidDefinitionException: Cannot construct instance of `java.time.Instant` (no Creators, like default construct, exist): cannot deserialize from Object value (no delegate- or property-based Creator)
 at [Source: (String)"{"createdOn":{"nano":68000000,"epochSecond":1580127683}

FYI:

I have created a configuration file where is register explicitly the JavaTimeModule to the Object Mapper

 @Configuration
public class JacksonConfiguration {

  @Value("${application.serialization.include-type-key:true}")
  private boolean includeTypeKey = true;

  @Bean
  public ObjectMapper objectMapper() {
    ObjectMapper mapper = new ObjectMapper();
    mapper.registerModule(new JavaTimeModule());
    mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
    mapper.addHandler(new MissingTypeIdHandler());
    if (includeTypeKey) {
      mapper.setMixInResolver(new TypeKeyMixInResolver());
    }
    return mapper;
  }

}

Need help!

Body:
Well if I'm not completely wrong, your mapping fails due to the wrong format. The json you get looks like this:

 "createdOn": {
    "nano": 877000000,
    "epochSecond": 1579861613
  },
  "changedOn": {
    "nano": 920000000,
    "epochSecond": 1579861613
  },

That means you have 2 objects 'createdOn' and 'changedOn' with two properties (nano, epochSecond), while you try to map it to one object containing two properties named 'createdOn' and 'changedOn'. You need to modify that, you have e.g. a class called Entry, with two properties (nano, epochSeconds) and then a class with two properties (createdOn, changedOn) of type Entry


问:
Create origin-Destination matrix from a Data Frame in Python
I would like to create an origin-destination matrix from the following data frame in python:

```<code>Origin  Destination
1         2
1         3
1         4
2         3
3         4
</code>```

I expect the following matrix:

```<code>   1  2  3  4
1  0  1  1  1
2  0  0  1  0
3  0  0  0  1
4  0  0  0  0
</code>```

I know that it could be done in R using table() function, but I don't know how to do it in python. Many thanks for any help.

答:
You could use pivot_table with and aggregate function of len to build the matrix:

```<code>df.pivot_table(values='Destination', index="Origin", columns='Destination',
           fill_value=0, aggfunc=len)
</code>```

which gives:

```<code>Destination  2  3  4
Origin
1            1  1  1
2            0  1  0
3            0  0  1
</code>```

But you will only find the origins and destination existing in the original matrix.

If you want a row and a column for every possible endpoint, you will have to first build an empty matrix and then add the above one:

```<code>resul = pd.DataFrame(0, index=list(range(1,5)), columns = list(range(1,5))
                ).add(df.pivot_table(values='Destination', index="Origin",
                                     columns='Destination', aggfunc=len),
                      fill_value=0).astype('int')
</code>```

which gives the expected matrix:

```<code>   1  2  3  4
1  0  1  1  1
2  0  0  1  0
3  0  0  0  1
4  0  0  0  0
</code>```

Body:
I would like to create an origin-destination matrix from the following data frame in python:

Origin  Destination
1         2
1         3
1         4
2         3
3         4

I expect the following matrix:

   1  2  3  4
1  0  1  1  1
2  0  0  1  0
3  0  0  0  1
4  0  0  0  0

I know that it could be done in R using table() function, but I don't know how to do it in python. Many thanks for any help.

Body:
You could use pivot_table with and aggregate function of len to build the matrix:

df.pivot_table(values='Destination', index="Origin", columns='Destination',
           fill_value=0, aggfunc=len)

which gives:

Destination  2  3  4
Origin
1            1  1  1
2            0  1  0
3            0  0  1

But you will only find the origins and destination existing in the original matrix.

If you want a row and a column for every possible endpoint, you will have to first build an empty matrix and then add the above one:

resul = pd.DataFrame(0, index=list(range(1,5)), columns = list(range(1,5))
                ).add(df.pivot_table(values='Destination', index="Origin",
                                     columns='Destination', aggfunc=len),
                      fill_value=0).astype('int')

which gives the expected matrix:

   1  2  3  4
1  0  1  1  1
2  0  0  1  0
3  0  0  0  1
4  0  0  0  0